Skip to main content

runner_manager_agent/
lifecycle.rs

1// owner: e3-jit-lifecycle-recovery
2
3//! One ephemeral runner, from an allocation decision to a scrubbed runtime.
4//!
5//! The ordering in this module is intentional.  An attempt is written before
6//! the package or GitHub is touched, the JIT value exists only in a restrictive
7//! handoff, and a registration-timeout termination is journalled before the
8//! process is signalled.  Recovery uses the same code as ordinary supervision;
9//! startup merely supplies the first observation.
10
11#[cfg(test)]
12use std::collections::VecDeque;
13use std::collections::{BTreeMap, BTreeSet, HashMap};
14use std::ffi::OsStr;
15use std::fmt;
16use std::fs;
17use std::io::Write;
18use std::num::NonZeroU16;
19use std::path::{Path, PathBuf};
20use std::sync::{Arc, Mutex};
21use std::time::Duration;
22
23use async_trait::async_trait;
24use runner_manager_domain::attempt::{
25    AttemptOutcome, AttemptState, FailureReason, GithubRunnerObservation, IsolationProviderFailure,
26    RecoveryDecision, RecoveryObservation, RecoveryTimeouts, RunnerAttempt, authorize,
27    recovery_decision,
28};
29use runner_manager_domain::execution::{AttemptExecution, Backend, ImageReference};
30use runner_manager_domain::model::{AttemptId, Clock, HostId, PolicyId, ScaleTarget};
31use runner_manager_domain::path::LocalAbsolutePath;
32use runner_manager_domain::policy::ScalePolicy;
33use runner_manager_domain::store::{Store, StoreError};
34use runner_manager_domain::workspace::{AttemptWorkspace, WorkspacePolicy};
35use runner_manager_github::jit::{
36    DEFAULT_WORK_FOLDER, EncodedJitConfig, JitError, JitGateway, JitRegistration, JitRunnerRequest,
37};
38use runner_manager_github::rest::{CancelToken, InventoryGateway};
39use runner_manager_platform::process::{
40    Adoption, ChildProcess, ProcessIdentity, RestrictiveHandoff, SpawnSpec, Termination,
41};
42use runner_manager_platform::runner_account::{RunnerAccount, RunnerPool};
43use runner_manager_platform::runner_env::{self, RunnerEnv, RunnerPlatform};
44use runner_manager_platform::runner_root::{
45    self, RootOwner, RootPreflight, RunnerRootError, default_runner_root,
46};
47use secrecy::SecretString;
48
49use crate::dependency_caches::DependencyCaches;
50use crate::oci::OciProcesses;
51use crate::package::{
52    InstalledPackage, LastCheck, PackageCache, PackageContents, PackageError, RunnerVersion,
53};
54use crate::reconcile::{
55    AllocationGuard, CleanBackoff, EventSink, LaunchFailure, LaunchRequest, LifecycleEvent,
56    OutcomeKind, ReplacementIntent, ReservationWaits, RunnerLauncher, failure_reason_kind,
57};
58
59mod windows_hyperv;
60pub use windows_hyperv::{WindowsHyperVContainers, WindowsHyperVHostState};
61
62const IDENTITY_FILE: &str = ".runner-process.json";
63const FALLBACK_IDENTITY_FILE: &str = ".runner-process.recovery.json";
64const UNRESOLVED_PROCESS_FILE: &str = ".runner-process.unresolved";
65const RUNNER_ID_FILE: &str = ".github-runner-id";
66const TERMINATE_INTENT_FILE: &str = ".terminate-registration-timeout";
67const MAX_POST_SPAWN_STOP_ATTEMPTS: usize = 3;
68
69/// Slot-root names a cleaned persistent attempt must not have left behind.
70///
71/// This is not the rule — the rule is that *nothing* but a real `_work`
72/// survives, and [`verify_slot_scrubbed`] enforces that by counting. This list
73/// is the second, independent question asked of the same directory: each name
74/// is stat-ed directly, so a scrub that skipped one is caught even if the
75/// enumeration that was supposed to find it under-reported. Every entry is one
76/// of the things `04-security-recovery.md` requires to be proven absent before a
77/// slot is released; the encoded JIT handoff is the one exception, matched by
78/// its published prefix in [`verify_slot_scrubbed`] because the rest of its name
79/// is a UUID. Being compile-time constants, these are also the only entry names
80/// a refusal message is allowed to print.
81const SENSITIVE_SLOT_ENTRIES: &[&str] = &[
82    // Runner binaries and the launchers beside them.
83    "bin",
84    "externals",
85    "run.sh",
86    "run.cmd",
87    "config.sh",
88    "config.cmd",
89    // The registration identity GitHub's runner writes for itself, and the
90    // per-run environment it reads back.
91    ".runner",
92    ".credentials",
93    ".credentials_rsaparams",
94    ".env",
95    ".path",
96    "_diag",
97    // This agent's own process-identity and lifecycle sidecars.
98    IDENTITY_FILE,
99    FALLBACK_IDENTITY_FILE,
100    UNRESOLVED_PROCESS_FILE,
101    RUNNER_ID_FILE,
102    TERMINATE_INTENT_FILE,
103];
104#[cfg(test)]
105const TEST_LISTENER_READY: &str = ".test-listener-ready";
106
107/// GitHub Runner v2.336.0 accepts JIT configuration for `run` through its
108/// secret `ACTIONS_RUNNER_INPUT_JITCONFIG` input. The platform spawn boundary
109/// supplies that input from the restrictive handoff; the listener command line
110/// must contain only the supported `run` command.
111///
112/// The environment is the daemon's plus [`runner_env::platform_defaults`], then
113/// the dependency caches (`caches`, from [`DependencyCaches::for_launch`]), then
114/// the host's `runner.env`, which wins over either of the others.
115/// The per-attempt temporary directory goes last and `runner.env` cannot name
116/// it. On macOS a runner that still starts throttled is reported
117/// ([`SpawnSpec::expect_normal_scheduling`]).
118fn runner_listener_spec(
119    program: PathBuf,
120    runtime: &Path,
121    caches: Vec<(&'static str, std::ffi::OsString)>,
122    host_env: &RunnerEnv,
123) -> SpawnSpec {
124    let tmp = runtime.join("tmp");
125    let _ = std::fs::create_dir_all(&tmp);
126    let platform = RunnerPlatform::current();
127    let defaults = runner_env::platform_defaults(
128        platform,
129        runtime,
130        &runner_env::Inherited::current(),
131        Path::exists,
132    );
133    // The Windows profile, or the Linux home, the defaults point at. Created
134    // even when `runner.env` overrides part of it: it is inside the attempt, so
135    // it costs nothing and goes with the attempt.
136    for dir in runner_env::directories_to_create(platform, &defaults) {
137        let _ = std::fs::create_dir_all(dir);
138    }
139    let mut spec = SpawnSpec::new(program)
140        .arg("run")
141        .working_dir(runtime)
142        .expect_normal_scheduling();
143    for (name, value) in runner_env::runner_environment(defaults, caches, host_env) {
144        spec = spec.env(name, value);
145    }
146    spec.env("TMPDIR", &tmp).env("TEMP", &tmp).env("TMP", &tmp)
147}
148
149/// Where in an attempt the Docker proxy listens when the runner runs as another
150/// account: `state/` is the daemon's alone, so the socket has to be somewhere
151/// the runner can reach.
152const DOCKER_SOCKET_DIR: &str = ".docker-proxy";
153
154/// Hands `runtime` to `account`, and shares every cache directory `spec`
155/// names with the whole `pool`.
156///
157/// The attempt is the account's alone: the runner writes its configuration,
158/// `_diag` and `_work` beside its own binaries. A cache is the opposite, the
159/// one thing runners are meant to share, so it goes to the pool's group
160/// instead. A cache directory already shared is not walked again.
161fn hand_over(
162    account: RunnerAccount,
163    pool: &RunnerPool,
164    runtime: &Path,
165    spec: &SpawnSpec,
166    cache_names: &[&str],
167) -> std::io::Result<()> {
168    hand_over_attempt(account, runtime)?;
169    for name in cache_names {
170        let Some(value) = spec.configured_env(name) else {
171            continue;
172        };
173        let path = Path::new(value);
174        if !path.is_absolute() {
175            continue;
176        }
177        match std::fs::symlink_metadata(path) {
178            Ok(metadata) if metadata.is_dir() && !pool.is_shared(path) => {
179                pool.share_tree(path)?;
180            }
181            Ok(_) => {}
182            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
183            Err(error) => return Err(error),
184        }
185    }
186    Ok(())
187}
188
189/// Hands the attempt directory at `runtime` to `account`, and lets the pool's
190/// group into the runner root above it.
191fn hand_over_attempt(account: RunnerAccount, runtime: &Path) -> std::io::Result<()> {
192    account.give_tree(runtime)?;
193    #[cfg(unix)]
194    {
195        use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _};
196        // The attempt is its account's alone; no other account, another
197        // runner's included, may enter it. Its parent, the runner root, the
198        // pool's group may read and pass through: root owns it, so no job can
199        // rename or replace another job's attempt. Read as well as search,
200        // because `Runner.Listener` refuses to start unless it can list every
201        // directory above its own.
202        fs::set_permissions(runtime, fs::Permissions::from_mode(0o700))?;
203        if let Some(root) = runtime.parent() {
204            let metadata = fs::symlink_metadata(root)?;
205            if metadata.is_dir() && metadata.uid() == 0 {
206                if metadata.gid() != account.gid {
207                    std::os::unix::fs::lchown(root, None, Some(account.gid))?;
208                }
209                if metadata.mode() & 0o7777 != 0o750 {
210                    fs::set_permissions(root, fs::Permissions::from_mode(0o750))?;
211                }
212            }
213        }
214    }
215    Ok(())
216}
217
218/// The pool account that owns `path`, if any.
219fn owner_in_pool(pool: &RunnerPool, path: &Path) -> Option<RunnerAccount> {
220    #[cfg(unix)]
221    {
222        use std::os::unix::fs::MetadataExt as _;
223        let metadata = fs::symlink_metadata(path).ok()?;
224        pool.account(metadata.uid()).filter(|_| metadata.is_dir())
225    }
226    #[cfg(not(unix))]
227    {
228        let _ = (pool, path);
229        None
230    }
231}
232
233/// What an attempt's process-identity records name.
234#[derive(Debug)]
235enum RecordedProcess {
236    /// Nothing is recorded.
237    None,
238    /// A process the runner account runs, or one that has exited.
239    Runner(ProcessIdentity),
240    /// A live process the runner account does not run: not this daemon's
241    /// runner, so never signalled, and treated as gone.
242    Foreign,
243}
244
245impl RecordedProcess {
246    fn runner(self) -> Option<ProcessIdentity> {
247        match self {
248            Self::Runner(identity) => Some(identity),
249            Self::None | Self::Foreign => None,
250        }
251    }
252}
253
254/// Who a native runner runs as.
255#[derive(Debug, Clone, Default)]
256enum RunnerIdentity {
257    /// As the daemon itself.
258    #[default]
259    Daemon,
260    /// As an account of this pool, its own for the life of its attempt.
261    Pool(RunnerPool),
262    /// As nobody: a confined `root` without the pool starts no runner, and
263    /// this says why.
264    Missing(String),
265}
266
267impl RunnerIdentity {
268    /// The pool runners run as, `None` for the daemon's own account, or why no
269    /// runner may start.
270    fn pool(&self) -> Result<Option<&RunnerPool>, FailureReason> {
271        match self {
272            Self::Daemon => Ok(None),
273            Self::Pool(pool) => Ok(Some(pool)),
274            Self::Missing(why) => Err(FailureReason::Other(why.clone())),
275        }
276    }
277}
278
279/// Retry bounds for failures that can resolve without operator action.
280#[derive(Debug, Clone, Copy, PartialEq, Eq)]
281pub struct RetryPolicy {
282    pub max_attempts: u32,
283    pub initial: Duration,
284    pub maximum: Duration,
285}
286
287impl RetryPolicy {
288    #[must_use]
289    pub const fn bounded(max_attempts: u32, initial: Duration, maximum: Duration) -> Self {
290        Self {
291            max_attempts,
292            initial,
293            maximum,
294        }
295    }
296
297    pub(crate) fn delay(self, failure_index: u32) -> Duration {
298        let shift = failure_index.saturating_sub(1).min(31);
299        self.initial
300            .saturating_mul(1_u32 << shift)
301            .min(self.maximum)
302    }
303}
304
305/// Non-secret lifecycle evidence.  Payloads are identifiers and closed enums;
306/// neither the encoded configuration nor child output can enter this type.
307#[derive(Debug, Clone, PartialEq, Eq)]
308pub enum AttemptEvent {
309    State {
310        attempt: AttemptId,
311        state: AttemptState,
312    },
313    Retry {
314        attempt: AttemptId,
315        operation: &'static str,
316        delay: Duration,
317    },
318    /// How long each step of a launch took, whether or not it started a
319    /// runner, so a slow or failed start can be attributed to the wait for the
320    /// allocation lock, the package copy, the JIT request, or the process start.
321    Launched {
322        attempt: AttemptId,
323        started: bool,
324        timings: LaunchTimings,
325    },
326    Adopted {
327        attempt: AttemptId,
328    },
329    RemoteIdentityRecovered {
330        attempt: AttemptId,
331        runner_id: u64,
332    },
333    TerminateIntent {
334        attempt: AttemptId,
335    },
336    Terminated {
337        attempt: AttemptId,
338    },
339    /// The attempt's GitHub registration was removed by this agent. Carries the
340    /// runner id because that is the identifier an operator sees in the
341    /// target's runner settings, and the attempt id is not shown there.
342    Deregistered {
343        attempt: AttemptId,
344        runner_id: u64,
345    },
346    Concluded {
347        attempt: AttemptId,
348        outcome: OutcomeKind,
349    },
350    Cleaned {
351        attempt: AttemptId,
352        outcome: OutcomeKind,
353    },
354    QuarantinedOwnedResource {
355        attempt: AttemptId,
356    },
357}
358
359pub trait AttemptEventSink: fmt::Debug + Send + Sync {
360    fn emit(&self, event: AttemptEvent);
361}
362
363#[derive(Debug, Default)]
364pub struct AttemptEventLog(Mutex<Vec<AttemptEvent>>);
365
366impl AttemptEventLog {
367    #[must_use]
368    pub fn events(&self) -> Vec<AttemptEvent> {
369        self.0
370            .lock()
371            .map(|events| events.clone())
372            .unwrap_or_default()
373    }
374}
375
376impl AttemptEventSink for AttemptEventLog {
377    fn emit(&self, event: AttemptEvent) {
378        if let Ok(mut events) = self.0.lock() {
379            events.push(event);
380        }
381    }
382}
383
384#[derive(Debug, Clone, Copy, Default)]
385pub struct NoAttemptEvents;
386
387impl AttemptEventSink for NoAttemptEvents {
388    fn emit(&self, _event: AttemptEvent) {}
389}
390
391/// Where one launch spent its time. A step the launch never reached is zero.
392#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
393pub struct LaunchTimings {
394    /// Waiting for older launches of this host to reserve first.
395    pub order_wait: Duration,
396    /// Waiting for the host allocation lock, which another launch may hold, and
397    /// for the rest of its pass to be reserved after it.
398    pub lock_wait: Duration,
399    /// Waiting for a turn to prepare: see [`preparation_turns`].
400    pub turn_wait: Duration,
401    pub materialize: Duration,
402    /// Where `materialize` went.
403    pub package: PackageTimings,
404    /// The execution provider's preparation: its environment, and the checks
405    /// of the host's runner environment and dependency caches.
406    pub prepare: Duration,
407    pub register: Duration,
408    pub spawn: Duration,
409}
410
411impl LaunchTimings {
412    #[must_use]
413    pub fn total(&self) -> Duration {
414        self.order_wait
415            + self.lock_wait
416            + self.turn_wait
417            + self.materialize
418            + self.prepare
419            + self.register
420            + self.spawn
421    }
422}
423
424/// Where a launch's package step went, across its retries.
425#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
426pub struct PackageTimings {
427    /// Waiting for another launch's install, or for copies an install must
428    /// not run under.
429    pub wait: Duration,
430    /// Checking the catalog, and downloading and extracting a package.
431    pub install: Duration,
432    /// Fingerprinting the trees a copy may trust.
433    pub verify: Duration,
434    /// Copying or cloning the tree into the runner's runtime.
435    pub copy: Duration,
436    /// A package was downloaded for this launch.
437    pub downloaded: bool,
438}
439
440#[derive(Debug, Clone, Copy)]
441enum LaunchStep {
442    Turn,
443    Materialize,
444    Prepare,
445    Register,
446    Spawn,
447}
448
449/// Times a launch's steps; beginning one ends the one before, so an early
450/// return still leaves the step it failed in measured once [`Self::finish`]
451/// runs.
452#[derive(Debug)]
453struct LaunchStopwatch {
454    timings: LaunchTimings,
455    running: Option<(LaunchStep, std::time::Instant)>,
456}
457
458impl LaunchStopwatch {
459    fn new(waits: ReservationWaits) -> Self {
460        Self {
461            timings: LaunchTimings {
462                order_wait: waits.order,
463                lock_wait: waits.lock,
464                ..LaunchTimings::default()
465            },
466            running: None,
467        }
468    }
469
470    fn begin(&mut self, step: LaunchStep) {
471        self.finish();
472        self.running = Some((step, std::time::Instant::now()));
473    }
474
475    fn finish(&mut self) {
476        if let Some((step, since)) = self.running.take() {
477            let slot = match step {
478                LaunchStep::Turn => &mut self.timings.turn_wait,
479                LaunchStep::Materialize => &mut self.timings.materialize,
480                LaunchStep::Prepare => &mut self.timings.prepare,
481                LaunchStep::Register => &mut self.timings.register,
482                LaunchStep::Spawn => &mut self.timings.spawn,
483            };
484            *slot = since.elapsed();
485        }
486    }
487}
488
489/// A launch slower than this is logged as a warning, with its steps.
490///
491/// A healthy launch is a few seconds; a queued job waits for all of it, and
492/// the daemon's default filter drops the `info` line that times every launch.
493const SLOW_LAUNCH: Duration = Duration::from_secs(60);
494
495/// The daemon's sink for retries and launch timings; the other attempt events
496/// already reach the log through the reconcile sink.
497///
498/// Every field name is on `d1`'s allow-list, or the redacting layer would log
499/// `[redacted]` in its place; `tests::every_field_name_the_attempt_sink_emits_is_allowed`
500/// keeps that true.
501#[derive(Debug, Clone, Copy, Default)]
502pub struct TracingAttemptEvents;
503
504impl AttemptEventSink for TracingAttemptEvents {
505    fn emit(&self, event: AttemptEvent) {
506        match event {
507            AttemptEvent::Retry {
508                attempt,
509                operation,
510                delay,
511            } => tracing::warn!(
512                event = "attempt_retry",
513                attempt_id = %attempt,
514                reason = operation,
515                retry_in_ms = millis(delay),
516            ),
517            AttemptEvent::Launched {
518                attempt,
519                started,
520                timings,
521            } => {
522                macro_rules! launched {
523                    ($level:ident, $name:literal) => {
524                        tracing::$level!(
525                            event = $name,
526                            attempt_id = %attempt,
527                            duration_ms = millis(timings.total()),
528                            order_wait_ms = millis(timings.order_wait),
529                            lock_wait_ms = millis(timings.lock_wait),
530                            turn_wait_ms = millis(timings.turn_wait),
531                            materialize_ms = millis(timings.materialize),
532                            package_wait_ms = millis(timings.package.wait),
533                            package_install_ms = millis(timings.package.install),
534                            package_verify_ms = millis(timings.package.verify),
535                            package_copy_ms = millis(timings.package.copy),
536                            package_downloaded = timings.package.downloaded,
537                            prepare_ms = millis(timings.prepare),
538                            register_ms = millis(timings.register),
539                            spawn_ms = millis(timings.spawn),
540                        )
541                    };
542                }
543                if !started {
544                    launched!(warn, "attempt_launch_failed");
545                } else if timings.total() >= SLOW_LAUNCH {
546                    launched!(warn, "attempt_launch_slow");
547                } else {
548                    launched!(info, "attempt_launched");
549                }
550            }
551            _ => {}
552        }
553    }
554}
555
556fn millis(duration: Duration) -> u64 {
557    u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
558}
559
560/// Whether the demand that justified a retry still exists.
561#[async_trait]
562pub trait DemandPersistence: fmt::Debug + Send + Sync {
563    async fn persists(&self, policy: PolicyId) -> bool;
564}
565
566#[derive(Debug, Clone, Copy, Default)]
567pub struct PersistentDemand;
568
569#[async_trait]
570impl DemandPersistence for PersistentDemand {
571    async fn persists(&self, _policy: PolicyId) -> bool {
572        true
573    }
574}
575
576#[async_trait]
577pub trait RetryDelay: fmt::Debug + Send + Sync {
578    async fn wait(&self, duration: Duration);
579}
580
581#[derive(Debug, Clone, Copy, Default)]
582pub struct TokioRetryDelay;
583
584#[async_trait]
585impl RetryDelay for TokioRetryDelay {
586    async fn wait(&self, duration: Duration) {
587        tokio::time::sleep(duration).await;
588    }
589}
590
591#[derive(Debug, Clone, PartialEq, Eq)]
592pub struct JitRequestFailure {
593    pub terminal: bool,
594    pub reason: FailureReason,
595    pub retry_after: Option<Duration>,
596}
597
598/// GitHub's authoritative runner state plus the identity returned by inventory.
599/// The id is carried independently of the local sidecar so recovery can close
600/// the crash boundary immediately after a successful remote registration.
601#[derive(Debug, Clone, Copy, PartialEq, Eq)]
602pub struct LifecycleGithubObservation {
603    pub status: GithubRunnerObservation,
604    pub runner_id: Option<u64>,
605}
606
607impl LifecycleGithubObservation {
608    #[must_use]
609    pub const fn unreachable() -> Self {
610        Self {
611            status: GithubRunnerObservation::Unreachable,
612            runner_id: None,
613        }
614    }
615
616    #[must_use]
617    pub const fn not_registered() -> Self {
618        Self {
619            status: GithubRunnerObservation::NotRegistered,
620            runner_id: None,
621        }
622    }
623
624    #[must_use]
625    pub const fn registered(runner_id: u64, busy: bool) -> Self {
626        Self {
627            status: GithubRunnerObservation::Registered { busy },
628            runner_id: Some(runner_id),
629        }
630    }
631}
632
633/// The two GitHub views the lifecycle needs, combined so one fake can drive
634/// registration and authoritative runner telemetry.
635#[async_trait]
636pub trait LifecycleGithub: fmt::Debug + Send + Sync {
637    async fn register(
638        &self,
639        target: &ScaleTarget,
640        request: &JitRunnerRequest,
641        cancel: &CancelToken,
642    ) -> Result<JitRegistration, JitRequestFailure>;
643
644    async fn observe(
645        &self,
646        target: &ScaleTarget,
647        attempt: AttemptId,
648        cancel: &CancelToken,
649    ) -> LifecycleGithubObservation;
650
651    /// Remove one runner registration this agent created.
652    ///
653    /// Answers whether the registration is gone, and is deliberately not
654    /// fallible in the `Result` sense: no caller may abandon a conclusion
655    /// because GitHub was unreachable. See
656    /// [`LifecycleLauncher::deregister_runner`].
657    async fn deregister(&self, target: &ScaleTarget, runner_id: u64, cancel: &CancelToken) -> bool;
658}
659
660#[async_trait]
661impl<T> LifecycleGithub for T
662where
663    T: JitGateway + InventoryGateway + fmt::Debug + Send + Sync,
664{
665    async fn register(
666        &self,
667        target: &ScaleTarget,
668        request: &JitRunnerRequest,
669        cancel: &CancelToken,
670    ) -> Result<JitRegistration, JitRequestFailure> {
671        self.generate_jit_config(target, request, cancel)
672            .await
673            .map_err(|error| {
674                let reason = if matches!(&error, JitError::Forbidden { .. }) {
675                    FailureReason::Other(
676                        "GitHub refused JIT registration with 403; check the App's runner permission and runner-group access"
677                            .into(),
678                    )
679                } else {
680                    FailureReason::JitRequestFailed
681                };
682                JitRequestFailure {
683                    terminal: error.is_terminal(),
684                    reason,
685                    retry_after: error
686                        .rate_limited()
687                        .map(|limit| limit.delay_from(self.now())),
688                }
689            })
690    }
691
692    async fn observe(
693        &self,
694        target: &ScaleTarget,
695        attempt: AttemptId,
696        cancel: &CancelToken,
697    ) -> LifecycleGithubObservation {
698        let expected_name = runner_name(attempt);
699        match self.list_runners(target, cancel).await {
700            Ok(inventory) => inventory
701                .runners()
702                .iter()
703                .find(|runner| runner.name == expected_name)
704                .map_or(LifecycleGithubObservation::not_registered(), |runner| {
705                    LifecycleGithubObservation::registered(runner.id, runner.busy)
706                }),
707            Err(_) => LifecycleGithubObservation::unreachable(),
708        }
709    }
710
711    async fn deregister(&self, target: &ScaleTarget, runner_id: u64, cancel: &CancelToken) -> bool {
712        self.remove_runner(target, runner_id, cancel).await.is_ok()
713    }
714}
715
716/// Package/cache operations used by one attempt.
717#[async_trait]
718pub trait RuntimePackages: fmt::Debug + Send + Sync {
719    /// Copy the package into `attempt`'s runtime, adding where the time went
720    /// to `timings`.
721    async fn materialize(
722        &self,
723        attempt: &RunnerAttempt,
724        timings: &mut PackageTimings,
725    ) -> Result<RunnerVersion, FailureReason>;
726    fn release(&self, attempt: AttemptId) -> Result<(), FailureReason>;
727    fn prune_obsolete_guarded(
728        &self,
729        authority: PruneAuthority<'_>,
730        attempts: &[RunnerAttempt],
731    ) -> Result<(), FailureReason>;
732}
733
734/// Unforgeable evidence that pruning was reached through e1's launch request.
735/// The type is public only because it appears in the public adapter trait; its
736/// private field and constructor prevent callers from substituting a guard
737/// acquired from an unrelated lock.
738pub struct PruneAuthority<'a> {
739    _guard: &'a AllocationGuard,
740}
741
742impl fmt::Debug for PruneAuthority<'_> {
743    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
744        f.write_str("PruneAuthority")
745    }
746}
747
748impl<'a> PruneAuthority<'a> {
749    fn from_launch_request(guard: &'a AllocationGuard) -> Self {
750        Self { _guard: guard }
751    }
752
753    #[cfg(test)]
754    pub(crate) const fn for_test(guard: &'a AllocationGuard) -> Self {
755        Self { _guard: guard }
756    }
757}
758
759/// What every target of one daemon shares about the one package directory they
760/// all use: which trees this process vouches for, and who may touch them.
761///
762/// Per target, one target's reinstall replaces the tree another target
763/// recorded, whose next launch finds it changed and reinstalls in turn: on a
764/// host serving two targets, every switch between them was a full download.
765#[derive(Debug, Clone, Default)]
766pub struct SharedPackages {
767    /// Held shared to copy from a tree, exclusive to change one. A cache entry
768    /// or seed is not replaced under a copy.
769    trees: Arc<tokio::sync::RwLock<()>>,
770    trusted: Arc<TrustedTrees>,
771    last_check: LastCheck,
772    /// The last install that failed, and when: the launches queued behind it
773    /// get its answer. See [`CachedRuntimePackages::materialize`].
774    failed_install: Arc<Mutex<Option<(std::time::Instant, FailureReason)>>>,
775}
776
777impl SharedPackages {
778    /// Stops trusting a shared package, unless a later install has replaced the
779    /// record this launch read: that one is another package's. The next launch
780    /// prepares it again.
781    fn discard(
782        &self,
783        root: &Path,
784        origin: &Path,
785        contents: &Arc<PackageContents>,
786        reason: &'static str,
787    ) {
788        if self.trusted.forget_if_holding(root, contents) {
789            tracing::warn!(
790                event = "package_verify_failed",
791                reason,
792                "a shared runner package does not match its verified archive; it is discarded \
793                 and prepared again"
794            );
795            self.trusted.forget(origin);
796        }
797    }
798
799    /// What an install ended in, for the launches queued behind it. A terminal
800    /// failure is not shared: it would close each of them, with no retry of its
801    /// own, and one that succeeded clears the last failure.
802    fn note_install(&self, result: Result<(), &FailureReason>) {
803        *self
804            .failed_install
805            .lock()
806            .unwrap_or_else(std::sync::PoisonError::into_inner) = result
807            .err()
808            .filter(|reason| !package_failure_is_terminal(reason))
809            .map(|reason| (std::time::Instant::now(), reason.clone()));
810    }
811
812    /// What an install that failed while a launch queued since `queued` for it
813    /// answered.
814    fn install_failed_since(&self, queued: std::time::Instant) -> Option<FailureReason> {
815        self.failed_install
816            .lock()
817            .unwrap_or_else(std::sync::PoisonError::into_inner)
818            .as_ref()
819            .filter(|(at, _)| *at >= queued)
820            .map(|(_, reason)| reason.clone())
821    }
822
823    /// For each target's [`PackageCache`]: the published version is one fact,
824    /// so one target's check spares the others theirs, and each check holds
825    /// every copy on this host back.
826    #[must_use]
827    pub fn last_check(&self) -> LastCheck {
828        self.last_check.clone()
829    }
830}
831
832#[cfg(test)]
833impl SharedPackages {
834    pub(crate) async fn copying(&self) -> tokio::sync::RwLockReadGuard<'_, ()> {
835        self.trees.read().await
836    }
837
838    /// Trusts `root`'s look as it is now, keeping what it must hold: what an edit
839    /// no fingerprint can see leaves behind, as on Windows, or by a superuser.
840    pub(crate) fn blind_to_edits_of(&self, root: &Path) {
841        let contents = self.trusted.contents_of(root);
842        self.trusted.record(root, contents).expect("recorded");
843    }
844
845    pub(crate) fn vouches_for(&self, root: &Path) -> bool {
846        self.trusted.contents_of(root).is_some()
847    }
848}
849
850/// Production package adapter.  It takes an e2 lease before returning, so a
851/// cache entry can never look unused while its runtime is starting.
852#[derive(Debug)]
853pub struct CachedRuntimePackages {
854    cache: Arc<PackageCache>,
855    shared: SharedPackages,
856}
857
858/// Add how long `work` took to `slot`.
859async fn timed<T>(slot: &mut Duration, work: impl std::future::Future<Output = T>) -> T {
860    let began = std::time::Instant::now();
861    let result = work.await;
862    *slot += began.elapsed();
863    result
864}
865
866impl CachedRuntimePackages {
867    /// Run `work` against a package tree on the blocking pool: a fingerprint
868    /// walks ~9,000 entries and a copy can take minutes.
869    async fn blocking<T: Send + 'static>(
870        &self,
871        root: &Path,
872        work: impl FnOnce(&TrustedTrees, &Path) -> T + Send + 'static,
873    ) -> Result<T, FailureReason> {
874        let trusted = Arc::clone(&self.shared.trusted);
875        let root = root.to_path_buf();
876        tokio::task::spawn_blocking(move || work(&trusted, &root))
877            .await
878            .map_err(|_| FailureReason::ProcessStartFailed)
879    }
880
881    /// One target's view of the package directory the targets of `shared` use.
882    #[must_use]
883    pub const fn new(cache: Arc<PackageCache>, shared: SharedPackages) -> Self {
884        Self { cache, shared }
885    }
886
887    /// A copy that cannot be used: removed, and the trees it came from read to see
888    /// whose fault it is. A runtime can be touched by a job, or sit on a disk with a
889    /// fault, and that is no reason to download anything. A tree that differs is
890    /// discarded, and prepared again by the next launch that needs it.
891    async fn reject_copy(
892        &self,
893        attempt: &RunnerAttempt,
894        root: &Path,
895        origin: &Path,
896        contents: &Arc<PackageContents>,
897        timings: &mut PackageTimings,
898    ) -> Result<(), FailureReason> {
899        let (attempt_copy, checking, from) =
900            (attempt.clone(), Arc::clone(contents), origin.to_path_buf());
901        let differs = timed(
902            &mut timings.verify,
903            self.blocking(root, move |_, source| {
904                let _ = remove_materialized_package(&attempt_copy);
905                let shared = checking.verify(source, |_| false);
906                shared.and_then(|()| {
907                    if from == source {
908                        Ok(())
909                    } else {
910                        checking.verify(&from, |_| false)
911                    }
912                })
913            }),
914        )
915        .await?;
916        if let Err(mismatch) = differs {
917            self.shared
918                .discard(root, origin, contents, mismatch.as_str());
919        }
920        Ok(())
921    }
922
923    /// The tree a runtime can be copied from right now, under nothing more than
924    /// the shared hold: nothing to check for or install, and a tree still as
925    /// this process left it.
926    async fn ready(
927        &self,
928        destination: &Path,
929        timings: &mut PackageTimings,
930    ) -> Result<Option<(InstalledPackage, PathBuf)>, FailureReason> {
931        let Some(installed) = self.cache.reusable().map_err(package_failure)? else {
932            return Ok(None);
933        };
934        let destination = destination.to_path_buf();
935        let verifying = self.blocking(installed.root(), move |trusted, source| {
936            if trusted.holds(source) {
937                copy_origin_ready(source, &destination, trusted)
938            } else {
939                None
940            }
941        });
942        let origin = timed(&mut timings.verify, verifying).await?;
943        Ok(origin.map(|origin| (installed, origin)))
944    }
945
946    /// Under the exclusive hold: a package installed and trusted, and the tree
947    /// to copy it from.
948    async fn install_trusted(
949        &self,
950        destination: &Path,
951        timings: &mut PackageTimings,
952    ) -> Result<(InstalledPackage, PathBuf), FailureReason> {
953        let (mut installed, downloaded) = timed(
954            &mut timings.install,
955            self.cache.ensure_installed_reporting(),
956        )
957        .await
958        .map_err(package_failure)?;
959        timings.downloaded |= downloaded;
960        // A cache entry this process did not install, or one that has changed
961        // since, may hold a job's edits; replace it with one downloaded and
962        // verified now. One this call just downloaded is trusted as it lands.
963        // See `TrustedTrees`.
964        let fresh = downloaded
965            || !timed(
966                &mut timings.verify,
967                self.blocking(installed.root(), TrustedTrees::holds),
968            )
969            .await?;
970        if fresh && !downloaded {
971            // After a restart there is simply no record. With one, this is an edit.
972            if self.shared.trusted.contents_of(installed.root()).is_some() {
973                tracing::warn!(
974                    event = "package_verify_failed",
975                    reason = "changed",
976                    "a shared runner package changed since it was prepared; it is discarded and \
977                     prepared again"
978                );
979            }
980            installed = timed(&mut timings.install, self.cache.reinstall())
981                .await
982                .map_err(package_failure)?;
983            timings.downloaded = true;
984        }
985        let destination = destination.to_path_buf();
986        let contents = installed.contents();
987        let origin = timed(
988            &mut timings.verify,
989            self.blocking(installed.root(), move |trusted, source| {
990                if fresh {
991                    trusted.record(source, contents)?;
992                }
993                Ok::<_, std::io::Error>(copy_origin(source, &destination, trusted))
994            }),
995        )
996        .await?
997        .map_err(|_| FailureReason::ProcessStartFailed)?;
998        Ok((installed, origin))
999    }
1000}
1001
1002#[async_trait]
1003impl RuntimePackages for CachedRuntimePackages {
1004    async fn materialize(
1005        &self,
1006        attempt: &RunnerAttempt,
1007        timings: &mut PackageTimings,
1008    ) -> Result<RunnerVersion, FailureReason> {
1009        let destination = attempt.runtime_path().to_path_buf();
1010        let trees = &self.shared.trees;
1011        // Only a launch that has something to change goes exclusive, and it lets
1012        // go before it copies, so the launches queued behind it do not wait for
1013        // its copy too. Once more if the tree still is not ready: a job edited
1014        // it meanwhile, or a seed cannot be made here. Then it copies under what
1015        // it just made, which no other launch can change.
1016        let mut installs = 0;
1017        let (hold, (installed, origin)) = loop {
1018            let queued = std::time::Instant::now();
1019            let hold = timed(&mut timings.wait, trees.read()).await;
1020            if let Some(ready) = self.ready(&destination, timings).await? {
1021                break (hold, ready);
1022            }
1023            drop(hold);
1024            // An install that failed while this launch queued for it answers
1025            // this launch too. Trying again at once, one queued launch after
1026            // another, would hold every copy of the package back for as many
1027            // stalled downloads as there are launches; the launch's own retry,
1028            // after its back-off, may try the download.
1029            let exclusive = timed(&mut timings.wait, trees.write()).await;
1030            if let Some(reason) = self.shared.install_failed_since(queued) {
1031                return Err(reason);
1032            }
1033            let installed = self.install_trusted(&destination, timings).await;
1034            self.shared.note_install(installed.as_ref().map(|_| ()));
1035            let ready = installed?;
1036            installs += 1;
1037            if installs == 2 {
1038                break (exclusive.downgrade(), ready);
1039            }
1040        };
1041        // Taken under the hold, so it is the record of the tree this copies. None
1042        // vouches for a tree with no record: before it is copied, not after.
1043        let Some(contents) = self.shared.trusted.contents_of(installed.root()) else {
1044            self.shared.trusted.forget(installed.root());
1045            return Err(FailureReason::ProcessStartFailed);
1046        };
1047        let copied_from = origin.clone();
1048        // Off the async thread: the daemon runs a current-thread runtime, and a
1049        // copy can take minutes on a busy disk.
1050        let copied = timed(
1051            &mut timings.copy,
1052            self.blocking(installed.root(), {
1053                let destination = destination.clone();
1054                move |trusted, source| {
1055                    copy_package_from(source, &origin, &destination, trusted)?;
1056                    // Checked again after the copy: an edit a job made while it ran
1057                    // moved the entry's print, and the runtime may hold it.
1058                    if trusted.holds(source) {
1059                        Ok(())
1060                    } else {
1061                        Err(std::io::Error::other(
1062                            "the runner package changed while copied",
1063                        ))
1064                    }
1065                }
1066            }),
1067        )
1068        .await?;
1069        if copied.is_err() {
1070            // Nothing a failed or tainted copy left may reach a retry's copy,
1071            // which lays out over the same runtime.
1072            drop(hold);
1073            self.reject_copy(attempt, installed.root(), &copied_from, &contents, timings)
1074                .await?;
1075            return Err(FailureReason::ProcessStartFailed);
1076        }
1077        if let Err(error) = self.cache.lease(attempt, installed.version()) {
1078            // Undoing the copy must not undo the *job* workspace: a persistent
1079            // slot's `_work` is retained across attempts, and this rollback
1080            // runs before the attempt that would have owned it ever started.
1081            drop(hold);
1082            self.reject_copy(attempt, installed.root(), &copied_from, &contents, timings)
1083                .await?;
1084            return Err(package_failure(error));
1085        }
1086        // Leased while still holding the trees, so a prune that gets the
1087        // exclusive hold finds this runtime's lease.
1088        drop(hold);
1089        // What runs is this copy, so this copy is what is checked, whatever
1090        // happened to the shared trees meanwhile: read in full against the
1091        // archive, under neither the host lock nor the trees' hold.
1092        let persistent = matches!(attempt.workspace(), AttemptWorkspace::PersistentSlot { .. });
1093        let checking = Arc::clone(&contents);
1094        let verified = timed(
1095            &mut timings.verify,
1096            self.blocking(installed.root(), move |_, _| {
1097                checking.verify(&destination, |name| persistent && is_work_folder(name))
1098            }),
1099        )
1100        .await?;
1101        if let Err(mismatch) = verified {
1102            let _ = self.cache.release(attempt.id);
1103            tracing::warn!(
1104                event = "package_copy_differs",
1105                attempt_id = %attempt.id,
1106                reason = mismatch.as_str(),
1107                "a runner's package copy does not match the package's verified archive; it is \
1108                 being removed and will not be used"
1109            );
1110            self.reject_copy(attempt, installed.root(), &copied_from, &contents, timings)
1111                .await?;
1112            return Err(FailureReason::ProcessStartFailed);
1113        }
1114        Ok(installed.version().clone())
1115    }
1116
1117    fn release(&self, attempt: AttemptId) -> Result<(), FailureReason> {
1118        self.cache.release(attempt).map_err(package_failure)
1119    }
1120
1121    /// Everything but the newest version, when nothing is using the trees: a
1122    /// copy from an older version, or an install in progress, makes this a
1123    /// no-op, and a later launch prunes what it left, which costs only disk.
1124    fn prune_obsolete_guarded(
1125        &self,
1126        _authority: PruneAuthority<'_>,
1127        attempts: &[RunnerAttempt],
1128    ) -> Result<(), FailureReason> {
1129        let Ok(_exclusive) = self.shared.trees.try_write() else {
1130            return Ok(());
1131        };
1132        let Some(current) = self.cache.newest_installed().map_err(package_failure)? else {
1133            return Ok(());
1134        };
1135        for installed in self.cache.installed().map_err(package_failure)? {
1136            if installed.version() != current.version() {
1137                match self.cache.prune(installed.version(), attempts) {
1138                    Ok(()) => self.shared.trusted.forget(installed.root()),
1139                    Err(PackageError::VersionInUse { .. }) => {}
1140                    Err(error) => return Err(package_failure(error)),
1141                }
1142            }
1143        }
1144        Ok(())
1145    }
1146}
1147
1148fn package_failure(error: PackageError) -> FailureReason {
1149    error.failure_reason().unwrap_or(FailureReason::Other(
1150        "runner package cache operation failed".into(),
1151    ))
1152}
1153
1154fn package_failure_is_terminal(reason: &FailureReason) -> bool {
1155    matches!(
1156        reason,
1157        FailureReason::RunnerPackageUnverified | FailureReason::RunnerVersionRejected
1158    )
1159}
1160
1161/// How many hex characters of the attempt id name its workspace.
1162///
1163/// # Why this is not the whole identifier, and why the policy is not in the path
1164///
1165/// Windows refuses a path over `MAX_PATH`, and the runner writes deep inside
1166/// this directory: `_work/<repo>/<repo>/.git/objects/pack/pack-<40 hex>.keep`
1167/// is 100 characters on its own before the repository is named twice. The
1168/// layout used to add two full identifiers -- the policy's and the attempt's,
1169/// 74 characters between them -- and that was enough to put a real checkout
1170/// over the line. Measured, not guessed: this repository's own CI failed here
1171/// three times in a row at 264 characters against a limit of 260, with
1172/// `fatal: cannot write keep file ...: Filename too long`. A repository whose
1173/// name is ten characters longer would have missed by fourteen.
1174///
1175/// The policy identifier is simply redundant -- an attempt identifier is
1176/// unique on its own, and nothing reads the directory tree to find a policy's
1177/// attempts, because [`crate::lifecycle::LifecycleLauncher`] asks the journal.
1178/// Twelve hex characters of the attempt is 48 bits, which for the handful of
1179/// directories one host holds at once is not a collision anybody will see, and
1180/// the journal keeps the full identifier either way.
1181///
1182/// Together that is 61 characters returned to the repository name.
1183const WORKSPACE_NAME_LEN: usize = 12;
1184
1185/// The directory name for one attempt's workspace.
1186fn workspace_name(id: AttemptId) -> String {
1187    let full = id.to_string();
1188    full.chars()
1189        .filter(|c| *c != '-')
1190        .take(WORKSPACE_NAME_LEN)
1191        .collect()
1192}
1193
1194/// Where one attempt's files go, and which cleanup algorithm they are owed.
1195///
1196/// The pair travels together because journalling them apart is exactly the bug
1197/// `AttemptWorkspace` exists to prevent: a `runtime_path` under a persistent
1198/// root recorded as ephemeral would be removed whole, taking the retained job
1199/// workspace with it.
1200#[derive(Debug, Clone)]
1201struct Placement {
1202    runtime: PathBuf,
1203    workspace: AttemptWorkspace,
1204}
1205
1206/// A runner-root refusal, rendered for the operator who has to fix it.
1207///
1208/// `RunnerRootError`'s `Display` already names the path, the relation and the
1209/// remediation command, and none of its variants can carry a credential — they
1210/// are paths, and `03-migration-rollout.md` requires the remediation command to
1211/// reach the operator verbatim.
1212fn root_failure(error: RunnerRootError) -> LifecycleError {
1213    // Logged here because here is the last place this is known at all. A launch
1214    // refused by the runner root fails *before* `record_allocation`, so no
1215    // attempt row is ever written: `b2` has nothing to carry the failure on and
1216    // `g2` has nothing to show it from. The lifecycle event keeps only
1217    // `reason=other`, by `failure_reason_kind`'s rule that no free text may
1218    // reach an event -- which left the whole refusal reading
1219    // `runner_start_failed reason=other`, once per poll, naming nothing.
1220    //
1221    // What travels is the *kind* and not the sentence, and that is forced rather
1222    // than chosen: `crate::logging` redacts every field it does not allow-list
1223    // and then scrubs anything path-shaped out of the ones it does, so a
1224    // rendered `RunnerRootError` -- which is mostly paths -- reaches the log as
1225    // `[redacted]`. `error_kind` is allow-listed and `RunnerRootError::kind` is
1226    // a closed vocabulary that survives the scrub, so this names which of a
1227    // dozen causes the operator has. The paths and the remediation reach them
1228    // through the command line, which is not redacted.
1229    tracing::warn!(
1230        error_kind = error.kind(),
1231        "the runner root refused this launch, so no attempt was created; the host will \
1232         retry every poll until the cause is resolved. Re-running `host set-runtime-root` \
1233         with the same path re-runs this check and prints the directory and the \
1234         remediation in full"
1235    );
1236    LifecycleError::Failed(FailureReason::Other(error.to_string()))
1237}
1238
1239/// The lowest positive slot inside `ceiling` that no uncleaned attempt holds.
1240///
1241/// `leases` is the journal's answer to "which slots are leased"
1242/// (`Store::slot_leases_for_policy`), which deliberately includes a terminal
1243/// attempt whose cleanup has not finished: that attempt still owns its
1244/// directory, so its slot is not free even though it no longer counts against
1245/// host capacity. `None` means the ceiling is reached, which is a refusal and
1246/// not a reason to allocate `s(ceiling + 1)`.
1247fn lowest_free_slot(leases: &[RunnerAttempt], ceiling: NonZeroU16) -> Option<NonZeroU16> {
1248    let held: BTreeSet<u16> = leases
1249        .iter()
1250        .filter_map(|attempt| attempt.workspace().slot_number())
1251        .collect();
1252    (1..=ceiling.get())
1253        .find(|slot| !held.contains(slot))
1254        .and_then(NonZeroU16::new)
1255}
1256
1257/// Create `<root>/sN`, or prove that what is already there is a real directory.
1258///
1259/// A symlink, junction or reparse point standing where the slot should be is
1260/// refused rather than followed: it is the one thing that could put an attempt's
1261/// files outside the root the operator configured, and
1262/// `04-security-recovery.md` requires that case to fail closed rather than to
1263/// be repaired here.
1264fn create_or_validate_slot(slot: &Path) -> Result<(), LifecycleError> {
1265    match fs::symlink_metadata(slot) {
1266        // [`is_link_like`] and not `is_symlink`, so that this is the same
1267        // question cleanup asks in [`slot_is_present`]: a reparse tag the
1268        // standard library has no name for is refused here rather than
1269        // allocated into and then quarantined forever by a cleanup that will
1270        // not scrub it.
1271        Ok(metadata) if is_link_like(&metadata) => Err(slot_refusal(
1272            slot,
1273            "is a symbolic link, junction or other reparse point, which could place runner \
1274             files outside the configured root",
1275        )),
1276        Ok(metadata) if !metadata.is_dir() => Err(slot_refusal(slot, "is not a directory")),
1277        Ok(_) => Ok(()),
1278        Err(error) if error.kind() == std::io::ErrorKind::NotFound => fs::create_dir(slot)
1279            .map_err(|source| slot_refusal(slot, format!("could not be created: {source}"))),
1280        Err(source) => Err(slot_refusal(
1281            slot,
1282            format!("could not be inspected: {source}"),
1283        )),
1284    }
1285}
1286
1287/// Accept a slot for reuse only when it is empty or holds one real `_work`.
1288///
1289/// `02-target-architecture.md`: "Before materialization, a reusable slot must
1290/// contain only a valid real `_work` directory or be empty." Everything else —
1291/// a leftover `bin/`, a link-shaped `_work`, a stray file — is refused here
1292/// rather than cleaned, because deciding whether those bytes are safe is
1293/// cleanup's and recovery's job (`c3`), and quietly reusing them would hand one
1294/// repository's retained state to the next attempt without anybody choosing to.
1295///
1296/// The inspection is one level deep and uses `symlink_metadata`, so nothing is
1297/// followed while it is being judged.
1298fn accept_reusable_slot(slot: &Path) -> Result<(), LifecycleError> {
1299    let unreadable =
1300        |source: std::io::Error| slot_refusal(slot, format!("could not be read: {source}"));
1301    let entries = fs::read_dir(slot).map_err(unreadable)?;
1302    let mut refused: Vec<String> = Vec::new();
1303    for entry in entries {
1304        let entry = entry.map_err(unreadable)?;
1305        let name = entry.file_name();
1306        let metadata = fs::symlink_metadata(entry.path()).map_err(|source| {
1307            slot_refusal(
1308                slot,
1309                format!("entry {name:?} could not be inspected: {source}"),
1310            )
1311        })?;
1312        // The same predicate cleanup retains by, so a `_work` this accepts is
1313        // one [`scrub_slot_entries`] will keep rather than refuse: a link, a
1314        // junction or any other reparse point is not a job workspace to either
1315        // of them.
1316        if is_retainable_work_folder(&name, &metadata) {
1317            continue;
1318        }
1319        refused.push(name.to_string_lossy().into_owned());
1320    }
1321    if refused.is_empty() {
1322        return Ok(());
1323    }
1324    refused.sort();
1325    Err(slot_refusal(
1326        slot,
1327        format!(
1328            "holds {} that this attempt may not reuse: [{}]. A reusable slot is empty or holds \
1329             one real `{DEFAULT_WORK_FOLDER}` directory and nothing else; remove or move the \
1330             entries listed, or let cleanup and recovery resolve them",
1331            if refused.len() == 1 {
1332                "an entry"
1333            } else {
1334                "entries"
1335            },
1336            refused.join(", ")
1337        ),
1338    ))
1339}
1340
1341fn slot_refusal(slot: &Path, detail: impl fmt::Display) -> LifecycleError {
1342    LifecycleError::Failed(FailureReason::Other(format!(
1343        "the persistent slot {} {detail}",
1344        slot.display()
1345    )))
1346}
1347
1348/// Whether a directory entry names the retained job workspace.
1349///
1350/// The comparison folds case on Windows because the filesystem does: there
1351/// `_Work` and `_work` are one directory, so a case-sensitive test would let
1352/// [`scrub_slot_entries`] delete the very directory it exists to keep, let
1353/// [`accept_reusable_slot`] refuse a slot that holds nothing but a valid job
1354/// workspace, and let a package's top-level `_Work` merge itself into the
1355/// previous attempt's `_work`. Elsewhere the two names really are two
1356/// directories and only the exact one is the job workspace.
1357fn is_work_folder(name: &OsStr) -> bool {
1358    if cfg!(windows) {
1359        name.eq_ignore_ascii_case(DEFAULT_WORK_FOLDER)
1360    } else {
1361        name == OsStr::new(DEFAULT_WORK_FOLDER)
1362    }
1363}
1364
1365/// Whether the operating system would follow this entry somewhere else.
1366///
1367/// `FileType::is_symlink` is the whole answer on Unix. On Windows it is not:
1368/// the standard library reports only the symlink and mount-point reparse tags,
1369/// and the substitution this has to refuse is *any* reparse point standing
1370/// where a real directory should be. So the attribute bit is the test there,
1371/// and a tag the standard library has no name for fails closed with the two it
1372/// does.
1373fn is_link_like(metadata: &fs::Metadata) -> bool {
1374    if metadata.file_type().is_symlink() {
1375        return true;
1376    }
1377    #[cfg(windows)]
1378    {
1379        use std::os::windows::fs::MetadataExt;
1380
1381        const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x0000_0400;
1382        metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0
1383    }
1384    #[cfg(not(windows))]
1385    false
1386}
1387
1388/// Whether an entry is the retained job workspace and is safe to retain.
1389///
1390/// The two halves are one question. A `_work` that is a file, a symlink, a
1391/// junction or any other reparse point is not a job workspace, and it is also
1392/// the exact substitution a hostile workflow makes to send cleanup somewhere
1393/// else (`04-security-recovery.md`, "A workflow replaces `_work` with a
1394/// junction or symlink to escape cleanup").
1395fn is_retainable_work_folder(name: &OsStr, metadata: &fs::Metadata) -> bool {
1396    is_work_folder(name) && metadata.is_dir() && !is_link_like(metadata)
1397}
1398
1399/// Undo one package materialization, dispatching on the journalled workspace.
1400///
1401/// The ephemeral half is what this always did: the directory is the attempt's
1402/// alone, so it goes whole. The persistent half removes the copy and nothing
1403/// else, because the slot's `_work` predates this attempt and outlives it
1404/// (`02-target-architecture.md`, "Persistent repository").
1405fn remove_materialized_package(attempt: &RunnerAttempt) -> std::io::Result<()> {
1406    match attempt.workspace() {
1407        AttemptWorkspace::Ephemeral => remove_runtime_tree(attempt.runtime_path()),
1408        AttemptWorkspace::PersistentSlot { .. } => scrub_slot_entries(attempt.runtime_path())
1409            .map_err(|quarantine| std::io::Error::other(quarantine.to_string())),
1410    }
1411}
1412
1413/// Remove a disposable runner tree without opening workflow-created special files.
1414///
1415/// On Unix, `remove_dir_all` 1.0 can open a FIFO while walking a directory and
1416/// wait forever for its peer. The .NET runner routinely leaves diagnostic FIFOs
1417/// in its private `tmp`, so use the standard library's fd-relative Unix remover,
1418/// which unlinks non-directories without opening them. Keep the external remover
1419/// on Windows for its existing read-only and long-path handling; when it is
1420/// refused, [`unlink_reparse_points`] takes every link out of its way and it
1421/// runs once more.
1422fn remove_runtime_tree(path: &Path) -> std::io::Result<()> {
1423    #[cfg(windows)]
1424    {
1425        match remove_dir_all::remove_dir_all(path) {
1426            Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
1427                unlink_reparse_points(path);
1428                remove_dir_all::remove_dir_all(path)
1429            }
1430            other => other,
1431        }
1432    }
1433    #[cfg(not(windows))]
1434    {
1435        fs::remove_dir_all(path)
1436    }
1437}
1438
1439/// Remove every symlink, junction and other reparse point under `dir` as a
1440/// link, without ever descending into one.
1441///
1442/// WinINet creates `…\INetCache\Content.IE5` as a junction whose own DACL denies
1443/// `Everyone` the right to list it, and a job that points `USERPROFILE` or
1444/// `LOCALAPPDATA` into its workspace gets one there. Both `remove_dir_all` 1.0
1445/// and `std::fs::remove_dir_all` then fail on the whole tree with
1446/// `ERROR_ACCESS_DENIED`, so the attempt could never be cleaned. Removing the
1447/// link itself is a different operation (`RemoveDirectoryW` on the reparse
1448/// point) that the deny entry does not cover, and once the links are gone the
1449/// tree is ordinary.
1450///
1451/// Only walked after a refusal, so an ordinary tree is traversed once. The
1452/// metadata comes from the directory listing itself, which on Windows
1453/// describes a reparse point without opening it.
1454///
1455/// Best effort: a failure here is not reported, because
1456/// [`remove_runtime_tree`]'s second `remove_dir_all` runs next and its result
1457/// is the authority on whether the tree went.
1458#[cfg(windows)]
1459fn unlink_reparse_points(dir: &Path) {
1460    let Ok(entries) = fs::read_dir(dir) else {
1461        return;
1462    };
1463    for entry in entries.flatten() {
1464        let path = entry.path();
1465        let Ok(metadata) = entry.metadata() else {
1466            continue;
1467        };
1468        if is_link_like(&metadata) {
1469            let _ = fs::remove_dir(&path).or_else(|_| fs::remove_file(&path));
1470        } else if metadata.is_dir() {
1471            unlink_reparse_points(&path);
1472        }
1473    }
1474}
1475
1476// ---------------------------------------------------------------------------
1477// Persistent cleanup (`04-security-recovery.md`, "Safe path handling")
1478// ---------------------------------------------------------------------------
1479
1480/// Why a persistent slot could not be proven safe to scrub.
1481///
1482/// A closed set rather than a formatted string, for two reasons that point the
1483/// same way. [`LifecycleEvent::AttemptCleanFailed`] takes a `&'static str` for
1484/// exactly the reason [`crate::reconcile::failure_reason_kind`] documents —
1485/// free text is the one shape that can carry a credential past a field
1486/// allow-list. And the entries under a slot root are *workflow-controlled*: a
1487/// job that writes a file named after a secret would publish it through any
1488/// message that echoed a directory listing, which is why nothing here ever
1489/// renders an entry name that did not come from this module's own constants.
1490#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1491enum SlotRefusal {
1492    /// The journalled runtime path is not `<root>/sN` for the journalled slot.
1493    NotTheJournalledSlot,
1494    /// A policy that still exists names a different root than the journal does.
1495    PolicyRootDisagrees,
1496    /// The slot is not strictly inside its root once components resolve.
1497    Containment,
1498    /// The slot itself is a file, a link, or could not be inspected.
1499    SlotNotADirectory,
1500    /// The slot's direct entries could not be listed.
1501    Enumeration,
1502    /// `_work` is a file, a symlink, a junction or another reparse point.
1503    WorkNotADirectory,
1504    /// An entry that had to go could not be removed.
1505    Deletion,
1506    /// Something other than the job workspace survived removal.
1507    Residue,
1508}
1509
1510impl SlotRefusal {
1511    /// The event field: a fixed vocabulary, never operator or workflow text.
1512    const fn class(self) -> &'static str {
1513        match self {
1514            Self::NotTheJournalledSlot => "slot_path_is_not_the_journalled_slot",
1515            Self::PolicyRootDisagrees => "slot_root_disagrees_with_policy",
1516            Self::Containment => "slot_escapes_its_root",
1517            Self::SlotNotADirectory => "slot_is_not_a_directory",
1518            Self::Enumeration => "slot_could_not_be_enumerated",
1519            Self::WorkNotADirectory => "retained_work_is_not_a_directory",
1520            Self::Deletion => "slot_entry_could_not_be_removed",
1521            Self::Residue => "slot_still_holds_runner_state",
1522        }
1523    }
1524
1525    /// What the operator has to do, in one sentence and with no path in it.
1526    const fn remediation(self) -> &'static str {
1527        match self {
1528            Self::NotTheJournalledSlot | Self::PolicyRootDisagrees | Self::Containment => {
1529                "the attempt keeps its slot lease and nothing was removed; correct the \
1530                 repository's persistent workspace path, or remove the slot directory by hand \
1531                 once you have confirmed what is in it"
1532            }
1533            Self::SlotNotADirectory | Self::WorkNotADirectory => {
1534                "the attempt keeps its slot lease and nothing was removed; a job replaced the \
1535                 slot or its `_work` with a link, so inspect it before deleting anything and \
1536                 treat the retained workspace as untrusted"
1537            }
1538            Self::Enumeration | Self::Deletion | Self::Residue => {
1539                "the attempt keeps its slot lease and will be cleaned again on the next pass; \
1540                 release whatever is holding the files open, or remove the slot's contents by \
1541                 hand leaving only `_work`"
1542            }
1543        }
1544    }
1545}
1546
1547/// A refusal that leaves one persistent slot quarantined.
1548///
1549/// `detail` is redacted by construction: it may hold paths this product
1550/// configured, `std::io::ErrorKind` values, counts, and names drawn from
1551/// [`SENSITIVE_SLOT_ENTRIES`] — and nothing that came out of a directory
1552/// listing.
1553#[derive(Debug, Clone, PartialEq, Eq)]
1554struct SlotQuarantine {
1555    refusal: SlotRefusal,
1556    detail: String,
1557}
1558
1559impl SlotQuarantine {
1560    fn new(refusal: SlotRefusal, detail: impl Into<String>) -> Self {
1561        Self {
1562            refusal,
1563            detail: detail.into(),
1564        }
1565    }
1566}
1567
1568impl fmt::Display for SlotQuarantine {
1569    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1570        write!(f, "{}; {}", self.detail, self.refusal.remediation())
1571    }
1572}
1573
1574/// Steps 1 to 3: the journalled root, the journalled slot, and containment.
1575///
1576/// Everything comes from the two immutable allocation facts — the exact runtime
1577/// path and the slot number. Not from the filesystem: scanning a root to decide
1578/// which directories are "mine" is what invariant 6 forbids, and it is also
1579/// impossible for an attempt whose policy has since been deleted. `configured`
1580/// is therefore a *cross-check* and not a source. A policy that survives has to
1581/// agree; a policy that does not survive removes a check rather than the
1582/// ability to clean the directory the journal already names.
1583///
1584/// The name test comes before the parent is used, so `<root>/s1` for a journal
1585/// row that says `s2` — and anything at all that is not one `sN` component —
1586/// is refused before a root is derived from it.
1587fn verify_journalled_slot(
1588    runtime: &Path,
1589    slot: NonZeroU16,
1590    configured: Option<&LocalAbsolutePath>,
1591) -> Result<(), SlotQuarantine> {
1592    let mislaid = || {
1593        SlotQuarantine::new(
1594            SlotRefusal::NotTheJournalledSlot,
1595            format!(
1596                "the journalled runtime {} is not the slot s{slot} this attempt was allocated as",
1597                runtime.display()
1598            ),
1599        )
1600    };
1601    let local = |path: &Path| {
1602        path.to_str()
1603            .and_then(|raw| LocalAbsolutePath::new(raw).ok())
1604            .ok_or_else(mislaid)
1605    };
1606
1607    let runtime_path = local(runtime)?;
1608    let root = local(runtime.parent().ok_or_else(mislaid)?)?;
1609    // Containment's lexical half, by construction: `derive_child` accepts one
1610    // component, so the equality below can only hold when the journalled path
1611    // really is this root's `sN` and nothing else.
1612    // The directory name comes from the domain that allocation named it with,
1613    // never from a second `s{n}` spelled out here: a convention with two
1614    // spellings would let cleanup refuse every slot the allocator created.
1615    let name = AttemptWorkspace::persistent_slot(slot)
1616        .slot_directory_name()
1617        .expect("a persistent workspace names its slot directory");
1618    let derived = runner_root::derive_child(&root, &name).map_err(|_| mislaid())?;
1619    if derived != runtime_path {
1620        return Err(mislaid());
1621    }
1622    if let Some(configured) = configured
1623        && configured != &root
1624    {
1625        return Err(SlotQuarantine::new(
1626            SlotRefusal::PolicyRootDisagrees,
1627            format!(
1628                "the journalled slot {} is not under the repository's configured persistent root \
1629                 {}",
1630                runtime.display(),
1631                configured.as_str()
1632            ),
1633        ));
1634    }
1635    // And containment's canonical half, which is what a junction planted inside
1636    // the root between allocation and cleanup has to get past.
1637    runner_root::verify_containment(&root, &derived).map_err(|source| {
1638        SlotQuarantine::new(
1639            SlotRefusal::Containment,
1640            format!("the journalled slot is not inside the root it was allocated from: {source}"),
1641        )
1642    })
1643}
1644
1645/// Whether the slot is there to be scrubbed at all, before its entries are.
1646///
1647/// `Ok(false)` — the directory is gone — is not a refusal. There is nothing to
1648/// remove and nothing to prove absent, which is the same tolerance the
1649/// disposable arm has always had for a runtime that vanished under it.
1650///
1651/// A slot that is a file, a link, a junction or any other reparse point *is* a
1652/// refusal, and it is checked here rather than inside the enumeration so that
1653/// the reason an operator reads names the shape rather than reporting that a
1654/// directory could not be listed.
1655fn slot_is_present(slot: &Path) -> Result<bool, SlotQuarantine> {
1656    match fs::symlink_metadata(slot) {
1657        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
1658        Err(source) => Err(SlotQuarantine::new(
1659            SlotRefusal::SlotNotADirectory,
1660            format!(
1661                "the slot {} could not be inspected: {:?}",
1662                slot.display(),
1663                source.kind()
1664            ),
1665        )),
1666        Ok(metadata) if !metadata.is_dir() || is_link_like(&metadata) => Err(SlotQuarantine::new(
1667            SlotRefusal::SlotNotADirectory,
1668            format!(
1669                "the slot {} is a link or a file rather than a real directory",
1670                slot.display()
1671            ),
1672        )),
1673        Ok(_) => Ok(true),
1674    }
1675}
1676
1677/// Steps 4 to 6: list the slot's direct entries, keep one real `_work`, remove
1678/// every other one.
1679///
1680/// Every call is a literal filesystem API against a path built from the slot
1681/// root and one entry name — no glob, no shell string, no repository-controlled
1682/// fragment. Nothing is followed: `symlink_metadata` says what each entry *is*,
1683/// and a link-shaped entry is unlinked rather than descended into, so a junction
1684/// planted where `bin` used to be cannot take the deletion outside the slot.
1685///
1686/// A `_work` that is not a real directory is the one entry this refuses to act
1687/// on at all. Removing it could destroy an operator's data if the link were
1688/// theirs; keeping it would hand the next attempt a workspace pointing
1689/// anywhere. `04-security-recovery.md` requires that case to quarantine the
1690/// slot, so the whole scrub stops there with nothing removed after it.
1691fn scrub_slot_entries(slot: &Path) -> Result<(), SlotQuarantine> {
1692    let unreadable = |source: std::io::Error| {
1693        SlotQuarantine::new(
1694            SlotRefusal::Enumeration,
1695            format!(
1696                "the entries of {} could not be listed: {:?}",
1697                slot.display(),
1698                source.kind()
1699            ),
1700        )
1701    };
1702    for entry in fs::read_dir(slot).map_err(unreadable)? {
1703        let name = entry.map_err(unreadable)?.file_name();
1704        let path = slot.join(&name);
1705        // Nothing is assumed from an entry that vanished: `verify_slot_scrubbed`
1706        // asks the filesystem again afterwards and refuses if it is still there.
1707        let Some(metadata) = listed_entry_metadata(&path).map_err(unreadable)? else {
1708            continue;
1709        };
1710        if is_work_folder(&name) {
1711            if is_retainable_work_folder(&name, &metadata) {
1712                continue;
1713            }
1714            return Err(SlotQuarantine::new(
1715                SlotRefusal::WorkNotADirectory,
1716                format!(
1717                    "the retained `{DEFAULT_WORK_FOLDER}` in {} is a link or a file rather than a \
1718                     real directory",
1719                    slot.display()
1720                ),
1721            ));
1722        }
1723        remove_slot_entry(&path, &metadata).map_err(|source| {
1724            SlotQuarantine::new(
1725                SlotRefusal::Deletion,
1726                format!(
1727                    "an entry of {} could not be removed: {:?}",
1728                    slot.display(),
1729                    source.kind()
1730                ),
1731            )
1732        })?;
1733    }
1734    Ok(())
1735}
1736
1737/// What a listed entry *is*, or `None` when it is no longer there.
1738///
1739/// An entry named by a listing and gone by the time it is stat-ed is absent,
1740/// which is a fact both passes over a slot want rather than an enumeration that
1741/// failed. Nothing is followed: `symlink_metadata` reports a link as a link.
1742fn listed_entry_metadata(path: &Path) -> std::io::Result<Option<fs::Metadata>> {
1743    match fs::symlink_metadata(path) {
1744        Ok(metadata) => Ok(Some(metadata)),
1745        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
1746        Err(source) => Err(source),
1747    }
1748}
1749
1750/// Remove one slot entry without following it.
1751///
1752/// An entry that is already gone is removed: the post-condition this serves is
1753/// "not there", and racing with whatever removed it first is not a refusal.
1754fn remove_slot_entry(path: &Path, metadata: &fs::Metadata) -> std::io::Result<()> {
1755    let removed = if is_link_like(metadata) {
1756        // A file symlink unlinks with `remove_file`; a directory symlink or a
1757        // Windows junction needs `remove_dir`. Neither follows the link.
1758        fs::remove_file(path).or_else(|_| fs::remove_dir(path))
1759    } else if metadata.is_dir() {
1760        remove_runtime_tree(path)
1761    } else {
1762        fs::remove_file(path)
1763    };
1764    match removed {
1765        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
1766        other => other,
1767    }
1768}
1769
1770/// Step 7: prove that nothing but the retained job workspace is left.
1771///
1772/// Two passes, deliberately independent, because the interesting failure is an
1773/// enumeration that under-reports. The first asks the directory what remains
1774/// and counts everything that is not a real `_work`. The second ignores the
1775/// listing entirely and stats the names an attempt is known to write — the
1776/// runner's binaries, the registration identity it stores beside them, the
1777/// process-identity sidecars and this agent's lifecycle marks — so a scrub that
1778/// silently skipped one is caught by a question that never consulted the
1779/// listing that skipped it.
1780///
1781/// Only the second pass names anything, and the encoded JIT handoff is reported
1782/// by its published prefix rather than by the UUID that follows it. A slot root
1783/// is workflow-writable, so a job that named a file after a secret would publish
1784/// it through any message that echoed the listing; the first pass therefore
1785/// reports a count.
1786fn verify_slot_scrubbed(slot: &Path) -> Result<(), SlotQuarantine> {
1787    let unreadable = |source: std::io::Error| {
1788        SlotQuarantine::new(
1789            SlotRefusal::Enumeration,
1790            format!(
1791                "the entries of {} could not be listed to verify the scrub: {:?}",
1792                slot.display(),
1793                source.kind()
1794            ),
1795        )
1796    };
1797    let mut residue = 0_usize;
1798    let mut named: Vec<String> = Vec::new();
1799    for entry in fs::read_dir(slot).map_err(unreadable)? {
1800        let name = entry.map_err(unreadable)?.file_name();
1801        // Absent is what this pass is here to establish. The named half below
1802        // stats every sensitive entry again, so an entry that only *looks*
1803        // absent to this listing is still caught.
1804        let Some(metadata) = listed_entry_metadata(&slot.join(&name)).map_err(unreadable)? else {
1805            continue;
1806        };
1807        if is_retainable_work_folder(&name, &metadata) {
1808            continue;
1809        }
1810        residue = residue.saturating_add(1);
1811        if name
1812            .to_string_lossy()
1813            .starts_with(RestrictiveHandoff::NAME_PREFIX)
1814        {
1815            named.push("an encoded JIT handoff".to_owned());
1816        }
1817    }
1818    named.extend(
1819        SENSITIVE_SLOT_ENTRIES
1820            .iter()
1821            .filter(|entry| fs::symlink_metadata(slot.join(entry)).is_ok())
1822            .map(|entry| format!("`{entry}`")),
1823    );
1824    if residue == 0 && named.is_empty() {
1825        return Ok(());
1826    }
1827    named.sort_unstable();
1828    named.dedup();
1829    Err(SlotQuarantine::new(
1830        SlotRefusal::Residue,
1831        residue_detail(slot, residue, &named),
1832    ))
1833}
1834
1835/// Word a [`SlotRefusal::Residue`] refusal from the two facts that produced it.
1836///
1837/// Separate from [`verify_slot_scrubbed`] because the disagreement it has to
1838/// report — a listing that counted nothing and a filesystem that answered
1839/// otherwise — is a race no test can stage, and a message that contradicts
1840/// itself is exactly what an operator reads at three in the morning.
1841///
1842/// `named` is the sanitized half: entries this crate published the names of.
1843/// A name a workflow chose is only ever counted, never echoed.
1844fn residue_detail(slot: &Path, residue: usize, named: &[String]) -> String {
1845    if residue == 0 {
1846        // The whole reason the second pass ignores the listing: the listing
1847        // reported a clean slot and the filesystem disagrees. Saying "0
1848        // entries survived" here would report the under-count as the fact.
1849        format!(
1850            "the listing of {} reported nothing but `{DEFAULT_WORK_FOLDER}`, yet {} survived \
1851                 cleanup",
1852            slot.display(),
1853            named.join(", ")
1854        )
1855    } else {
1856        format!(
1857            "{residue} entr{} other than `{DEFAULT_WORK_FOLDER}` survived cleanup of {}{}",
1858            if residue == 1 { "y" } else { "ies" },
1859            slot.display(),
1860            if named.is_empty() {
1861                String::new()
1862            } else {
1863                format!(", including {}", named.join(", "))
1864            }
1865        )
1866    }
1867}
1868
1869fn replacement_operation(outcome: &AttemptOutcome) -> Option<&'static str> {
1870    match outcome {
1871        AttemptOutcome::Failed {
1872            reason: FailureReason::JitExpired,
1873        } => Some("jit_expired_replacement"),
1874        AttemptOutcome::Failed {
1875            reason: FailureReason::ProcessExitedUnexpectedly,
1876        } => Some("exit_before_acceptance_replacement"),
1877        _ => None,
1878    }
1879}
1880
1881/// Lay the verified runner package out *around* whatever the slot retains.
1882///
1883/// `02-target-architecture.md`: "The verified runner package is copied into the
1884/// slot for the attempt", beside a `_work` that survives every attempt. Two
1885/// properties make that safe, and both are structural rather than documented:
1886///
1887/// * the walk is of the **source** tree, so a retained `_work` in the
1888///   destination is never opened, never descended into, and cannot be followed
1889///   wherever it might point;
1890/// * a top-level source entry named `_work` is refused rather than copied, so a
1891///   package that ever grew one could not merge itself into, or replace, the
1892///   job workspace of the attempt before it. The refusal is top-level only,
1893///   because the retained directory is a direct child of the slot; a `_work`
1894///   nested inside the package's own tree is an ordinary name.
1895///
1896/// On Unix the copy is a copy-on-write clone where the filesystem offers one,
1897/// taken from a [`package_seed`] on the destination's own volume. A plain
1898/// byte copy of the ~9,000-file package across volumes took four minutes on a
1899/// host whose runner root shared a busy disk with a build, and a queued job
1900/// waits for every second of it.
1901/// The tree to copy `source` from, making a seed where one helps; a change to
1902/// a tree, so only under the exclusive hold of [`SharedPackages::trees`].
1903///
1904/// A seed that cannot be made is a slower launch, not a failed one. The seed
1905/// sits under the runner root, beside job runtimes, so the refusal is re-checked
1906/// on it; a seed that fails it is bypassed.
1907fn copy_origin(source: &Path, destination: &Path, seeds: &TrustedTrees) -> PathBuf {
1908    #[cfg(unix)]
1909    {
1910        package_seed(source, destination, seeds)
1911            .ok()
1912            .filter(|seed| refuse_work_folder(seed).is_ok())
1913            .unwrap_or_else(|| source.to_path_buf())
1914    }
1915    #[cfg(not(unix))]
1916    {
1917        let _ = (destination, seeds);
1918        source.to_path_buf()
1919    }
1920}
1921
1922/// The tree to copy `source` from if it needs no change: `None` is a seed to
1923/// rebuild first.
1924fn copy_origin_ready(source: &Path, destination: &Path, seeds: &TrustedTrees) -> Option<PathBuf> {
1925    #[cfg(unix)]
1926    {
1927        match seed_location(source, destination) {
1928            Ok(None) => Some(source.to_path_buf()),
1929            Ok(Some(seeds_root)) => held_seed(source, &seeds_root, seeds)
1930                .ok()
1931                .flatten()
1932                .filter(|seed| refuse_work_folder(seed).is_ok()),
1933            Err(_) => None,
1934        }
1935    }
1936    #[cfg(not(unix))]
1937    {
1938        let _ = (destination, seeds);
1939        Some(source.to_path_buf())
1940    }
1941}
1942
1943/// Copy the package at `source` into `destination`, from `origin`: itself or
1944/// its seed. Under the shared hold of [`SharedPackages::trees`].
1945fn copy_package_from(
1946    source: &Path,
1947    origin: &Path,
1948    destination: &Path,
1949    seeds: &TrustedTrees,
1950) -> std::io::Result<()> {
1951    refuse_work_folder(source)?;
1952
1953    #[cfg(unix)]
1954    {
1955        clone_tree(origin, destination)?;
1956        // A seed a job edited during the clone no longer holds; the caller
1957        // re-checks the cache entry itself.
1958        if origin != source && !seeds.holds(origin) {
1959            return Err(std::io::Error::other(
1960                "the package seed changed while cloned",
1961            ));
1962        }
1963        Ok(())
1964    }
1965    #[cfg(not(unix))]
1966    {
1967        let _ = (origin, seeds);
1968        copy_package_entries(source, destination, true)
1969    }
1970}
1971
1972fn refuse_work_folder(source: &Path) -> std::io::Result<()> {
1973    if source.join(DEFAULT_WORK_FOLDER).exists() {
1974        return Err(std::io::Error::new(
1975            std::io::ErrorKind::InvalidData,
1976            "a cached runner package contains a _work folder, which means it was used \
1977             to run a job before it was archived; the cache must only contain clean \
1978             extracts to prevent data leakage",
1979        ));
1980    }
1981    Ok(())
1982}
1983
1984/// Where runtime copies of the package are cloned from, beside the runtimes.
1985#[cfg(unix)]
1986const PACKAGE_SEED_DIR: &str = ".runner-package";
1987
1988/// A copy of the cache entry on the destination's volume, so that every
1989/// runtime after the first is a same-volume clone rather than a cross-volume
1990/// byte copy. Returns `source` itself when it already shares the volume.
1991///
1992/// The seed lives at `<runner root>/.runner-package/<version>` and is replaced
1993/// whenever its manifest stops matching the cache entry's, which covers both a
1994/// new version and a reinstalled one. Superseded versions and interrupted
1995/// stagings are removed on that same pass. A seed is only made under the
1996/// exclusive hold of [`SharedPackages::trees`], so no launch is cloning from it
1997/// while it is replaced.
1998///
1999/// The seed sits one `..` from every job's runtime and is owned by the account
2000/// jobs run as, so it is only reused while it is provably the tree this
2001/// process built: see [`TrustedTrees`].
2002///
2003/// Only on macOS, where the runner root is all but certainly APFS and a clone
2004/// is free. On a Linux filesystem without reflinks a seed would only move every
2005/// copy's reads onto the runner root's own, possibly busy, disk.
2006#[cfg(unix)]
2007fn package_seed(
2008    source: &Path,
2009    destination: &Path,
2010    seeds: &TrustedTrees,
2011) -> std::io::Result<PathBuf> {
2012    match seed_location(source, destination)? {
2013        Some(root) => refresh_package_seed(source, &root, seeds),
2014        None => Ok(source.to_path_buf()),
2015    }
2016}
2017
2018/// Where `destination`'s seeds live, if copies to it use one.
2019#[cfg(unix)]
2020fn seed_location(source: &Path, destination: &Path) -> std::io::Result<Option<PathBuf>> {
2021    use std::os::unix::fs::MetadataExt as _;
2022
2023    if !cfg!(target_os = "macos") {
2024        return Ok(None);
2025    }
2026    let root = destination
2027        .parent()
2028        .ok_or_else(|| std::io::Error::other("a runtime directory has no parent"))?;
2029    if fs::metadata(source)?.dev() == fs::metadata(root)?.dev() {
2030        return Ok(None);
2031    }
2032    Ok(Some(root.join(PACKAGE_SEED_DIR)))
2033}
2034
2035/// The package trees this process built or installed: what each must hold, and
2036/// what each looked like right after. They are the cache entry runtimes are copied
2037/// from and the seeds they are cloned from.
2038///
2039/// Both are owned by the account jobs run as, so a job can edit any file in them,
2040/// and nothing on disk can say what they should hold: the published digest covers
2041/// the archive, not the extracted tree, and a record kept on disk could be
2042/// rewritten by the same account. So a tree without a record here, as every tree
2043/// has after a restart, is rebuilt before it is used.
2044///
2045/// Two checks, for two costs. [`PackageContents`] reads every file against the
2046/// verified archive, on every runtime a launch is about to run: that is the one
2047/// that decides. On Unix the tree's [`tree_fingerprint`] also spots a changed
2048/// tree cheaply before any of it is copied, since every edit, replacement or
2049/// rename moves the change time (`ctime`) of the entry or of its directory, and
2050/// only the superuser can set a change time.
2051#[derive(Debug, Default)]
2052pub(crate) struct TrustedTrees {
2053    trees: Mutex<HashMap<PathBuf, TrustedTree>>,
2054}
2055
2056#[derive(Debug, Clone)]
2057struct TrustedTree {
2058    #[cfg(unix)]
2059    print: [u8; 32],
2060    contents: Option<Arc<PackageContents>>,
2061}
2062
2063impl TrustedTrees {
2064    fn trees(&self) -> std::sync::MutexGuard<'_, HashMap<PathBuf, TrustedTree>> {
2065        self.trees
2066            .lock()
2067            .unwrap_or_else(std::sync::PoisonError::into_inner)
2068    }
2069
2070    /// Whether `root` is a tree recorded here and, where that can be told, still
2071    /// looks as it did.
2072    fn holds(&self, root: &Path) -> bool {
2073        let Some(tree) = self.trees().get(root).cloned() else {
2074            return false;
2075        };
2076        #[cfg(unix)]
2077        {
2078            tree_fingerprint(root).is_ok_and(|current| current == tree.print)
2079        }
2080        #[cfg(not(unix))]
2081        {
2082            let _ = tree;
2083            true
2084        }
2085    }
2086
2087    /// What `root` must hold.
2088    fn contents_of(&self, root: &Path) -> Option<Arc<PackageContents>> {
2089        self.trees()
2090            .get(root)
2091            .and_then(|tree| tree.contents.clone())
2092    }
2093
2094    /// Trust `root` as it is now; called right after this process made it.
2095    fn record(&self, root: &Path, contents: Option<Arc<PackageContents>>) -> std::io::Result<()> {
2096        let tree = TrustedTree {
2097            #[cfg(unix)]
2098            print: tree_fingerprint(root)?,
2099            contents,
2100        };
2101        self.trees().insert(root.to_path_buf(), tree);
2102        Ok(())
2103    }
2104
2105    /// Stops trusting `root` if `contents` is still what is recorded for it, and
2106    /// answers whether it did.
2107    fn forget_if_holding(&self, root: &Path, contents: &Arc<PackageContents>) -> bool {
2108        let mut trees = self.trees();
2109        let holding = trees
2110            .get(root)
2111            .and_then(|tree| tree.contents.as_ref())
2112            .is_some_and(|current| Arc::ptr_eq(current, contents));
2113        if holding {
2114            trees.remove(root);
2115        }
2116        holding
2117    }
2118
2119    /// Stop trusting `root`: its next use rebuilds it.
2120    fn forget(&self, root: &Path) {
2121        self.trees().remove(root);
2122    }
2123}
2124
2125/// A digest of every entry under `root`: its relative path, inode, size, mode,
2126/// owner and change time.
2127#[cfg(unix)]
2128fn tree_fingerprint(root: &Path) -> std::io::Result<[u8; 32]> {
2129    use sha2::{Digest as _, Sha256};
2130    use std::os::unix::fs::MetadataExt as _;
2131
2132    fn walk(root: &Path, path: &Path, hasher: &mut Sha256) -> std::io::Result<()> {
2133        let metadata = fs::symlink_metadata(path)?;
2134        let relative = path.strip_prefix(root).unwrap_or(path);
2135        hasher.update(relative.as_os_str().as_encoded_bytes());
2136        hasher.update([0]);
2137        for field in [
2138            metadata.ino(),
2139            metadata.size(),
2140            u64::from(metadata.mode()),
2141            u64::from(metadata.uid()),
2142            u64::from(metadata.gid()),
2143        ] {
2144            hasher.update(field.to_le_bytes());
2145        }
2146        hasher.update(metadata.ctime().to_le_bytes());
2147        hasher.update(metadata.ctime_nsec().to_le_bytes());
2148        if metadata.is_dir() {
2149            let mut children = fs::read_dir(path)?
2150                .map(|entry| entry.map(|entry| entry.path()))
2151                .collect::<std::io::Result<Vec<_>>>()?;
2152            children.sort();
2153            for child in children {
2154                walk(root, &child, hasher)?;
2155            }
2156        }
2157        Ok(())
2158    }
2159
2160    let mut hasher = Sha256::new();
2161    walk(root, root, &mut hasher)?;
2162    Ok(hasher.finalize().into())
2163}
2164
2165#[cfg(unix)]
2166fn seed_path(source: &Path, seeds: &Path) -> std::io::Result<PathBuf> {
2167    let version = source
2168        .file_name()
2169        .ok_or_else(|| std::io::Error::other("a cache entry has no name"))?;
2170    Ok(seeds.join(version))
2171}
2172
2173/// The seed of `source` under `seeds`, if its manifest matches the cache
2174/// entry's and its fingerprint the one this process recorded.
2175#[cfg(unix)]
2176fn held_seed(
2177    source: &Path,
2178    seeds: &Path,
2179    trusted: &TrustedTrees,
2180) -> std::io::Result<Option<PathBuf>> {
2181    use crate::package::MANIFEST_FILE;
2182
2183    let seed = seed_path(source, seeds)?;
2184    let manifest = fs::read(source.join(MANIFEST_FILE))?;
2185    let held = fs::read(seed.join(MANIFEST_FILE)).is_ok_and(|held| held == manifest)
2186        && trusted.holds(&seed);
2187    Ok(held.then_some(seed))
2188}
2189
2190/// The seed of `source` under `seeds`, rebuilt unless [`held_seed`] finds it.
2191#[cfg(unix)]
2192fn refresh_package_seed(
2193    source: &Path,
2194    seeds: &Path,
2195    trusted: &TrustedTrees,
2196) -> std::io::Result<PathBuf> {
2197    if let Some(seed) = held_seed(source, seeds, trusted)? {
2198        return Ok(seed);
2199    }
2200    let seed = seed_path(source, seeds)?;
2201
2202    // Superseded versions, a stale seed and interrupted stagings all go.
2203    match remove_runtime_tree(seeds) {
2204        Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error),
2205        _ => {}
2206    }
2207    fs::create_dir_all(seeds)?;
2208    let staging = seeds.join(format!(".staging-{}", uuid::Uuid::new_v4()));
2209    fs::create_dir(&staging)?;
2210    // A half-made staging would hold the disk space the fallback copy needs.
2211    if let Err(error) = clone_tree(source, &staging).and_then(|()| fs::rename(&staging, &seed)) {
2212        let _ = remove_runtime_tree(&staging);
2213        return Err(error);
2214    }
2215    // After the rename, which itself moves the seed directory's change time.
2216    trusted.record(&seed, None)?;
2217    Ok(seed)
2218}
2219
2220/// `cp -a`, asking for a copy-on-write clone where the platform's `cp` can.
2221///
2222/// macOS `cp -c` and GNU `--reflink=auto` both fall back to a byte copy across
2223/// volumes; a `cp` that does not know the flag at all gets a plain `cp -a`.
2224#[cfg(unix)]
2225fn clone_tree(source: &Path, destination: &Path) -> std::io::Result<()> {
2226    #[cfg(target_os = "macos")]
2227    const CLONE: &[&str] = &["-c"];
2228    #[cfg(target_os = "linux")]
2229    const CLONE: &[&str] = &["--reflink=auto"];
2230    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
2231    const CLONE: &[&str] = &[];
2232
2233    let from = format!("{}/.", source.display());
2234    let cp = |extra: &[&str]| {
2235        std::process::Command::new("cp")
2236            .arg("-a")
2237            .args(extra)
2238            .arg(&from)
2239            .arg(destination)
2240            .status()
2241    };
2242    if cp(CLONE)?.success() || cp(&[])?.success() {
2243        Ok(())
2244    } else {
2245        Err(std::io::Error::other("cp failed"))
2246    }
2247}
2248
2249#[cfg(not(unix))]
2250fn copy_package_entries(source: &Path, destination: &Path, top_level: bool) -> std::io::Result<()> {
2251    fs::create_dir_all(destination)?;
2252    for entry in fs::read_dir(source)? {
2253        let entry = entry?;
2254        if top_level && is_work_folder(&entry.file_name()) {
2255            return Err(std::io::Error::new(
2256                std::io::ErrorKind::InvalidData,
2257                format!(
2258                    "the runner package holds a top-level `{DEFAULT_WORK_FOLDER}`; copying \
2259                     it would overwrite the job workspace a persistent slot retains"
2260                ),
2261            ));
2262        }
2263        let target = destination.join(entry.file_name());
2264        if entry.file_type()?.is_dir() {
2265            copy_package_entries(&entry.path(), &target, false)?;
2266        } else {
2267            fs::copy(entry.path(), target)?;
2268        }
2269    }
2270    Ok(())
2271}
2272
2273/// Process operations are attempt-addressed so a recovered process and a child
2274/// started in this invocation are supervised through one port.
2275#[derive(Debug, Clone, PartialEq, Eq)]
2276pub struct ProcessStartFailure {
2277    pub reason: FailureReason,
2278    /// False once a child existed: the one-shot JIT value may have been
2279    /// consumed, so retrying it could start a duplicate.
2280    pub retryable: bool,
2281    /// Set only when cleanup could not prove the spawned process dead.  The
2282    /// caller must journal `starting` and retain capacity/supervision.
2283    pub live_pid: Option<u32>,
2284}
2285
2286impl ProcessStartFailure {
2287    pub(crate) fn before_spawn(reason: FailureReason) -> Self {
2288        Self {
2289            reason,
2290            retryable: true,
2291            live_pid: None,
2292        }
2293    }
2294
2295    fn after_spawn_stopped() -> Self {
2296        Self {
2297            reason: FailureReason::ProcessStartFailed,
2298            retryable: false,
2299            live_pid: None,
2300        }
2301    }
2302
2303    fn after_spawn_live(pid: u32) -> Self {
2304        Self::after_spawn_live_with_reason(pid, FailureReason::ProcessStartFailed)
2305    }
2306
2307    fn after_spawn_live_with_reason(pid: u32, reason: FailureReason) -> Self {
2308        Self {
2309            reason,
2310            retryable: false,
2311            live_pid: Some(pid),
2312        }
2313    }
2314}
2315
2316/// A credential-free capability report. An isolated policy never treats a
2317/// native-ready result as permission to launch.
2318#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2319pub enum ProviderCapability {
2320    Ready,
2321    Unsupported,
2322    NotInstalled,
2323    PermissionDenied,
2324    ImageUnavailableOrIncompatible,
2325    DiskQuotaUnavailable,
2326    Degraded,
2327}
2328
2329impl ProviderCapability {
2330    #[must_use]
2331    pub const fn display_name(self) -> &'static str {
2332        match self {
2333            Self::Ready => "ready",
2334            Self::Unsupported => "unsupported",
2335            Self::NotInstalled => "not installed",
2336            Self::PermissionDenied => "permission denied",
2337            Self::ImageUnavailableOrIncompatible => "image unavailable or incompatible",
2338            Self::DiskQuotaUnavailable => "disk quota unavailable",
2339            Self::Degraded => "degraded",
2340        }
2341    }
2342
2343    pub(crate) fn refusal(self) -> FailureReason {
2344        FailureReason::IsolationProvider(match self {
2345            Self::Ready | Self::Degraded => IsolationProviderFailure::Degraded,
2346            Self::Unsupported => IsolationProviderFailure::Unsupported,
2347            Self::NotInstalled => IsolationProviderFailure::NotInstalled,
2348            Self::PermissionDenied => IsolationProviderFailure::PermissionDenied,
2349            Self::ImageUnavailableOrIncompatible => {
2350                IsolationProviderFailure::ImageUnavailableOrIncompatible
2351            }
2352            Self::DiskQuotaUnavailable => IsolationProviderFailure::DiskQuotaUnavailable,
2353        })
2354    }
2355}
2356
2357/// Only the closed isolation category may cross from an adapter into the
2358/// attempt journal or operator error. Arbitrary adapter text could echo JIT.
2359fn safe_isolation_reason(
2360    reason: FailureReason,
2361    fallback: IsolationProviderFailure,
2362) -> FailureReason {
2363    match reason {
2364        FailureReason::IsolationProvider(_) => reason,
2365        _ => FailureReason::IsolationProvider(fallback),
2366    }
2367}
2368
2369#[derive(Debug, Clone, PartialEq, Eq)]
2370pub struct PreparedEnvironment {
2371    attempt: AttemptId,
2372    identity: Option<EnvironmentIdentity>,
2373    /// The target whose dependency-cache namespace a native runner uses:
2374    /// `prepare` sees the policy and `start` does not.
2375    cache_target: Option<runner_manager_domain::model::ScaleTarget>,
2376}
2377
2378impl PreparedEnvironment {
2379    /// An adapter must return the identity independently recorded on its resource.
2380    #[must_use]
2381    pub fn isolated(attempt: AttemptId, identity: EnvironmentIdentity) -> Self {
2382        Self {
2383            attempt,
2384            identity: Some(identity),
2385            cache_target: None,
2386        }
2387    }
2388
2389    #[must_use]
2390    pub fn identity(&self) -> Option<&EnvironmentIdentity> {
2391        self.identity.as_ref()
2392    }
2393}
2394
2395#[derive(Debug, Clone, PartialEq, Eq)]
2396pub struct ResolvedEnvironment {
2397    pub provider_kind: Backend,
2398    pub image: ImageReference,
2399}
2400
2401/// Closed, non-secret provider diagnostics; adapters cannot attach arbitrary output.
2402#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2403pub enum ProviderDiagnostic {
2404    CapabilityUnavailable,
2405    ImageRejected,
2406    PrepareFailed,
2407    StartFailed,
2408    OwnershipMismatch,
2409    CleanupDeferred,
2410}
2411
2412#[derive(Debug, Clone, PartialEq, Eq)]
2413pub enum EnvironmentIdentity {
2414    NativeProcess(u32),
2415    Isolated {
2416        host: HostId,
2417        attempt: AttemptId,
2418        provider_kind: Backend,
2419        environment_id: String,
2420        resolved_image: ImageReference,
2421        generation: String,
2422    },
2423}
2424
2425#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2426pub enum EnvironmentState {
2427    Starting,
2428    Running,
2429    Exited,
2430    Missing,
2431}
2432
2433#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2434pub enum ProviderStop {
2435    Stopped,
2436    StillRunning,
2437}
2438
2439#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2440pub enum ProviderDestroy {
2441    Destroyed,
2442    Deferred(&'static str),
2443}
2444
2445fn identity_matches_intent(
2446    host: HostId,
2447    attempt: &RunnerAttempt,
2448    identity: &EnvironmentIdentity,
2449) -> bool {
2450    identity_same_allocation(host, attempt, identity)
2451        && match (attempt.execution(), identity) {
2452            (
2453                AttemptExecution::Isolated { environment_id, .. },
2454                EnvironmentIdentity::Isolated {
2455                    environment_id: id, ..
2456                },
2457            ) => environment_id
2458                .as_ref()
2459                .is_none_or(|journal_id| journal_id == id),
2460            _ => false,
2461        }
2462}
2463
2464fn identity_same_allocation(
2465    host: HostId,
2466    attempt: &RunnerAttempt,
2467    identity: &EnvironmentIdentity,
2468) -> bool {
2469    match (attempt.execution(), identity) {
2470        (
2471            AttemptExecution::Isolated {
2472                provider_kind,
2473                environment_id: _,
2474                resolved_image,
2475                generation,
2476            },
2477            EnvironmentIdentity::Isolated {
2478                host: owner,
2479                attempt: owner_attempt,
2480                provider_kind: kind,
2481                environment_id: _,
2482                resolved_image: image,
2483                generation: resource_generation,
2484            },
2485        ) => {
2486            *owner == host
2487                && *owner_attempt == attempt.id
2488                && provider_kind == kind
2489                && resolved_image == image
2490                && generation == resource_generation
2491        }
2492        _ => false,
2493    }
2494}
2495
2496/// The JIT value can be passed into a provider start exactly once. It must never
2497/// become a provider identity, label, argument, persisted configuration or log field.
2498pub struct OneTimeJitHandoff<'a> {
2499    config: &'a EncodedJitConfig,
2500}
2501
2502impl fmt::Debug for OneTimeJitHandoff<'_> {
2503    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2504        f.write_str("OneTimeJitHandoff([redacted])")
2505    }
2506}
2507
2508impl<'a> OneTimeJitHandoff<'a> {
2509    pub(crate) fn new(config: &'a EncodedJitConfig) -> Self {
2510        Self { config }
2511    }
2512
2513    /// Consuming the handoff prevents the same token being delivered twice by one call.
2514    pub fn consume(self) -> &'a EncodedJitConfig {
2515        self.config
2516    }
2517}
2518
2519/// Provider lifecycle and native process supervision. The default lifecycle
2520/// methods adapt the existing native spawn/recovery implementation, and refuse
2521/// every isolated allocation. Future platform adapters must implement their
2522/// own prepare/start/inspect/stop/destroy paths rather than fall back here.
2523pub trait ExecutionProvider: fmt::Debug + Send + Sync {
2524    fn probe(&self, policy: &ScalePolicy) -> ProviderCapability {
2525        if policy.execution_policy().is_native() {
2526            ProviderCapability::Ready
2527        } else {
2528            ProviderCapability::Unsupported
2529        }
2530    }
2531
2532    /// Resolve Auto and image identity before allocation or GitHub JIT. Native has no image.
2533    fn resolve(&self, policy: &ScalePolicy) -> Result<Option<ResolvedEnvironment>, FailureReason> {
2534        if policy.execution_policy().is_native() {
2535            Ok(None)
2536        } else {
2537            Err(FailureReason::Other(
2538                "execution provider unavailable".into(),
2539            ))
2540        }
2541    }
2542
2543    fn prepare(
2544        &self,
2545        attempt: &RunnerAttempt,
2546        policy: &ScalePolicy,
2547    ) -> Result<PreparedEnvironment, FailureReason> {
2548        if !attempt.execution().is_native() || !policy.execution_policy().is_native() {
2549            return Err(FailureReason::Other(
2550                "execution provider unavailable".into(),
2551            ));
2552        }
2553        Ok(PreparedEnvironment {
2554            attempt: attempt.id,
2555            identity: None,
2556            cache_target: None,
2557        })
2558    }
2559
2560    fn start(
2561        &self,
2562        prepared: PreparedEnvironment,
2563        attempt: &RunnerAttempt,
2564        handoff: OneTimeJitHandoff<'_>,
2565    ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
2566        if prepared.attempt != attempt.id || !attempt.execution().is_native() {
2567            return Err(ProcessStartFailure::before_spawn(FailureReason::Other(
2568                "execution provider identity mismatch".into(),
2569            )));
2570        }
2571        self.spawn(attempt, handoff.consume())
2572            .map(EnvironmentIdentity::NativeProcess)
2573    }
2574
2575    fn inspect(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2576        if !attempt.execution().is_native() {
2577            return Err(FailureReason::Other(
2578                "execution provider unavailable".into(),
2579            ));
2580        }
2581        self.is_alive(attempt).map(|alive| {
2582            if alive {
2583                EnvironmentState::Running
2584            } else {
2585                EnvironmentState::Missing
2586            }
2587        })
2588    }
2589
2590    fn stop(&self, attempt: &RunnerAttempt) -> Result<ProviderStop, FailureReason> {
2591        if !attempt.execution().is_native() {
2592            return Err(FailureReason::Other(
2593                "execution provider unavailable".into(),
2594            ));
2595        }
2596        self.terminate(attempt).map(|()| ProviderStop::Stopped)
2597    }
2598
2599    fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
2600        if !attempt.execution().is_native() {
2601            return Err(FailureReason::Other(
2602                "execution provider unavailable".into(),
2603            ));
2604        }
2605        // Native termination is already settled by the existing recovery
2606        // decision before workspace cleanup. There is no separate sandbox
2607        // resource to destroy, and repeating an observation here changes the
2608        // established native cleanup behaviour.
2609        Ok(ProviderDestroy::Destroyed)
2610    }
2611
2612    fn recover(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2613        self.inspect(attempt)
2614    }
2615
2616    fn enumerate_owned(&self, _host_id: HostId) -> Vec<EnvironmentIdentity> {
2617        Vec::new() // Native processes have no independent provider resources.
2618    }
2619
2620    fn diagnostics(&self, _attempt: &RunnerAttempt) -> Vec<ProviderDiagnostic> {
2621        Vec::new() // No job or JIT output enters provider metadata.
2622    }
2623
2624    /// Independent resource metadata must match the journal before destructive calls.
2625    fn owns(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
2626        Ok(attempt.execution().is_native())
2627    }
2628
2629    fn spawn(
2630        &self,
2631        attempt: &RunnerAttempt,
2632        config: &EncodedJitConfig,
2633    ) -> Result<u32, ProcessStartFailure>;
2634    fn is_alive(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason>;
2635    /// Durable identity observed for a process that spawned before the
2636    /// `starting` journal write survived.
2637    fn recovered_pid(&self, attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason>;
2638    /// True only for a child this invocation owned and reaped with a successful
2639    /// exit status.  A recovered process that is merely gone answers false.
2640    fn completed_successfully(&self, attempt: &RunnerAttempt) -> bool;
2641    fn record_terminate_intent(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason>;
2642    fn has_terminate_intent(&self, attempt: &RunnerAttempt) -> bool;
2643    fn terminate(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason>;
2644}
2645
2646/// Compatibility name for existing native process tests and callers.
2647pub use ExecutionProvider as ProcessSupervisor;
2648
2649/// Native process supervision.  The start token is stored beside the runtime;
2650/// recovery never trusts a recycled PID merely because SQLite contains it.
2651#[derive(Debug, Default)]
2652pub struct NativeProcesses {
2653    children: Mutex<BTreeMap<AttemptId, ChildProcess>>,
2654    successful_exits: Mutex<BTreeMap<AttemptId, bool>>,
2655    /// The host's `runner.env`, read at every launch so that `host env set`
2656    /// reaches the next runner without a daemon restart. `None` applies none.
2657    runner_env_file: Option<PathBuf>,
2658    /// Persistent dependency caches. `None` gives runners none.
2659    dependency_caches: Option<DependencyCaches>,
2660    docker_cleanup: Option<crate::docker_cleanup::DockerCleanup>,
2661    /// Who runners run as. See [`runner_manager_platform::runner_account`].
2662    runner: RunnerIdentity,
2663    /// The pool account each attempt with a runner runs as.
2664    assignments: Mutex<BTreeMap<AttemptId, RunnerAccount>>,
2665    #[cfg(test)]
2666    post_spawn_faults: Mutex<VecDeque<PostSpawnBoundary>>,
2667    #[cfg(test)]
2668    post_spawn_reaps: std::sync::atomic::AtomicUsize,
2669    #[cfg(test)]
2670    post_spawn_stop_failures: std::sync::atomic::AtomicUsize,
2671    #[cfg(test)]
2672    use_long_lived_test_listener: std::sync::atomic::AtomicBool,
2673}
2674
2675/// Production provider set. Native and isolated allocations are routed by the
2676/// immutable execution kind in the policy/journal; an unavailable isolated
2677/// backend can never fall through to native process launch.
2678#[derive(Debug)]
2679pub struct PlatformExecutionProvider {
2680    native: NativeProcesses,
2681    oci: OciProcesses,
2682    windows_hyper_v: WindowsHyperVContainers,
2683}
2684
2685impl PlatformExecutionProvider {
2686    #[must_use]
2687    pub fn new(host_id: HostId) -> Self {
2688        Self {
2689            native: NativeProcesses::new(),
2690            oci: OciProcesses::new(host_id),
2691            windows_hyper_v: WindowsHyperVContainers::new(host_id),
2692        }
2693    }
2694
2695    /// Applies the operator's isolation settings to the OCI provider.
2696    #[must_use]
2697    pub fn with_isolation_settings(mut self, settings: crate::oci::IsolationSettings) -> Self {
2698        self.oci.apply(settings);
2699        self
2700    }
2701
2702    /// Applies the host's `runner.env` at `path` to every native runner.
2703    /// Isolated runners do not get it: their environment is the image's.
2704    #[must_use]
2705    pub fn with_runner_env_file(mut self, path: PathBuf) -> Self {
2706        self.native = self.native.with_runner_env_file(path);
2707        self
2708    }
2709
2710    /// Gives every native runner persistent dependency caches. Isolated
2711    /// runners do not get them: their providers mount no host directory.
2712    #[must_use]
2713    pub fn with_dependency_caches(mut self, caches: DependencyCaches) -> Self {
2714        self.native = self.native.with_dependency_caches(caches);
2715        self
2716    }
2717
2718    #[must_use]
2719    pub fn with_docker_cleanup(mut self, cleanup: crate::docker_cleanup::DockerCleanup) -> Self {
2720        self.native.docker_cleanup = Some(cleanup);
2721        self
2722    }
2723
2724    /// Starts every native runner as an account of `pool` of its own, which
2725    /// owns its attempt directory; the pool shares the caches. `None` keeps
2726    /// runners in the daemon's account.
2727    #[must_use]
2728    pub fn with_runner_pool(mut self, pool: Result<Option<RunnerPool>, String>) -> Self {
2729        self.native.runner = match pool {
2730            Ok(None) => RunnerIdentity::Daemon,
2731            Ok(Some(pool)) => {
2732                self.native.dependency_caches = self
2733                    .native
2734                    .dependency_caches
2735                    .take()
2736                    .map(DependencyCaches::without_git);
2737                RunnerIdentity::Pool(pool)
2738            }
2739            Err(why) => {
2740                tracing::error!(reason = "runner_account_missing", "{why}");
2741                RunnerIdentity::Missing(why)
2742            }
2743        };
2744        self
2745    }
2746
2747    fn isolated_provider(&self, backend: Backend) -> Option<&dyn ExecutionProvider> {
2748        match backend {
2749            Backend::Auto => {
2750                #[cfg(target_os = "windows")]
2751                {
2752                    Some(&self.windows_hyper_v)
2753                }
2754                #[cfg(not(target_os = "windows"))]
2755                {
2756                    Some(&self.oci)
2757                }
2758            }
2759            Backend::Oci => Some(&self.oci),
2760            Backend::WindowsHyperVContainer => Some(&self.windows_hyper_v),
2761            Backend::VirtualMachine => None,
2762        }
2763    }
2764
2765    fn provider_for_attempt(&self, attempt: &RunnerAttempt) -> Option<&dyn ExecutionProvider> {
2766        match attempt.execution() {
2767            AttemptExecution::Native { .. } => Some(&self.native),
2768            AttemptExecution::Isolated { provider_kind, .. } => {
2769                self.isolated_provider(*provider_kind)
2770            }
2771        }
2772    }
2773
2774    fn required_provider_for_attempt(
2775        &self,
2776        attempt: &RunnerAttempt,
2777    ) -> Result<&dyn ExecutionProvider, FailureReason> {
2778        self.provider_for_attempt(attempt)
2779            .ok_or_else(|| FailureReason::Other("execution provider unavailable".into()))
2780    }
2781}
2782
2783impl ExecutionProvider for PlatformExecutionProvider {
2784    fn probe(&self, policy: &ScalePolicy) -> ProviderCapability {
2785        match policy.execution_policy() {
2786            runner_manager_domain::execution::ExecutionPolicy::Native => self.native.probe(policy),
2787            runner_manager_domain::execution::ExecutionPolicy::Isolated { backend, .. } => self
2788                .isolated_provider(*backend)
2789                .map_or(ProviderCapability::Unsupported, |provider| {
2790                    provider.probe(policy)
2791                }),
2792        }
2793    }
2794
2795    fn resolve(&self, policy: &ScalePolicy) -> Result<Option<ResolvedEnvironment>, FailureReason> {
2796        match policy.execution_policy() {
2797            runner_manager_domain::execution::ExecutionPolicy::Native => {
2798                self.native.resolve(policy)
2799            }
2800            runner_manager_domain::execution::ExecutionPolicy::Isolated { backend, .. } => self
2801                .isolated_provider(*backend)
2802                .ok_or_else(|| FailureReason::Other("execution provider unavailable".into()))?
2803                .resolve(policy),
2804        }
2805    }
2806
2807    fn prepare(
2808        &self,
2809        attempt: &RunnerAttempt,
2810        policy: &ScalePolicy,
2811    ) -> Result<PreparedEnvironment, FailureReason> {
2812        match policy.execution_policy() {
2813            runner_manager_domain::execution::ExecutionPolicy::Native => {
2814                self.native.prepare(attempt, policy)
2815            }
2816            runner_manager_domain::execution::ExecutionPolicy::Isolated { backend, .. } => self
2817                .isolated_provider(*backend)
2818                .ok_or_else(|| FailureReason::Other("execution provider unavailable".into()))?
2819                .prepare(attempt, policy),
2820        }
2821    }
2822
2823    fn start(
2824        &self,
2825        prepared: PreparedEnvironment,
2826        attempt: &RunnerAttempt,
2827        handoff: OneTimeJitHandoff<'_>,
2828    ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
2829        self.provider_for_attempt(attempt)
2830            .ok_or_else(|| {
2831                ProcessStartFailure::before_spawn(FailureReason::Other(
2832                    "execution provider unavailable".into(),
2833                ))
2834            })?
2835            .start(prepared, attempt, handoff)
2836    }
2837
2838    fn inspect(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2839        self.required_provider_for_attempt(attempt)?
2840            .inspect(attempt)
2841    }
2842
2843    fn stop(&self, attempt: &RunnerAttempt) -> Result<ProviderStop, FailureReason> {
2844        self.required_provider_for_attempt(attempt)?.stop(attempt)
2845    }
2846
2847    fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
2848        self.required_provider_for_attempt(attempt)?
2849            .destroy(attempt)
2850    }
2851
2852    fn recover(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2853        self.required_provider_for_attempt(attempt)?
2854            .recover(attempt)
2855    }
2856
2857    fn enumerate_owned(&self, host_id: HostId) -> Vec<EnvironmentIdentity> {
2858        let mut owned = self.oci.enumerate_owned(host_id);
2859        owned.extend(self.windows_hyper_v.enumerate_owned(host_id));
2860        owned
2861    }
2862
2863    fn diagnostics(&self, attempt: &RunnerAttempt) -> Vec<ProviderDiagnostic> {
2864        self.provider_for_attempt(attempt)
2865            .map_or_else(Vec::new, |provider| provider.diagnostics(attempt))
2866    }
2867
2868    fn owns(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
2869        self.required_provider_for_attempt(attempt)?.owns(attempt)
2870    }
2871
2872    fn spawn(
2873        &self,
2874        attempt: &RunnerAttempt,
2875        config: &EncodedJitConfig,
2876    ) -> Result<u32, ProcessStartFailure> {
2877        if !attempt.execution().is_native() {
2878            return Err(ProcessStartFailure::before_spawn(FailureReason::Other(
2879                "native process launch refused for an isolated allocation".into(),
2880            )));
2881        }
2882        self.native.spawn(attempt, config)
2883    }
2884
2885    fn is_alive(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
2886        self.required_provider_for_attempt(attempt)?
2887            .is_alive(attempt)
2888    }
2889
2890    fn recovered_pid(&self, attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
2891        self.required_provider_for_attempt(attempt)?
2892            .recovered_pid(attempt)
2893    }
2894
2895    fn completed_successfully(&self, attempt: &RunnerAttempt) -> bool {
2896        self.provider_for_attempt(attempt)
2897            .is_some_and(|provider| provider.completed_successfully(attempt))
2898    }
2899
2900    fn record_terminate_intent(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
2901        self.required_provider_for_attempt(attempt)?
2902            .record_terminate_intent(attempt)
2903    }
2904
2905    fn has_terminate_intent(&self, attempt: &RunnerAttempt) -> bool {
2906        self.provider_for_attempt(attempt)
2907            .is_some_and(|provider| provider.has_terminate_intent(attempt))
2908    }
2909
2910    fn terminate(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
2911        self.required_provider_for_attempt(attempt)?
2912            .terminate(attempt)
2913    }
2914}
2915
2916#[cfg(test)]
2917#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2918enum PostSpawnBoundary {
2919    HandoffDelete,
2920    IdentitySerialize,
2921    IdentityWrite,
2922    ChildMapInsert,
2923}
2924
2925impl NativeProcesses {
2926    #[must_use]
2927    pub fn new() -> Self {
2928        Self::default()
2929    }
2930
2931    /// Applies the host's `runner.env` at `path` to every runner launched.
2932    #[must_use]
2933    pub fn with_runner_env_file(mut self, path: PathBuf) -> Self {
2934        self.runner_env_file = Some(path);
2935        self
2936    }
2937
2938    /// Gives every runner launched persistent dependency caches, without the
2939    /// Git checkout cache when runners run as another account.
2940    #[must_use]
2941    pub fn with_dependency_caches(mut self, caches: DependencyCaches) -> Self {
2942        self.dependency_caches = Some(match self.runner {
2943            RunnerIdentity::Pool(_) => caches.without_git(),
2944            RunnerIdentity::Daemon | RunnerIdentity::Missing(_) => caches,
2945        });
2946        self
2947    }
2948
2949    /// The cache variables for `attempt`, when `prepare` named its target.
2950    fn cache_env(
2951        &self,
2952        attempt: &RunnerAttempt,
2953        target: Option<&runner_manager_domain::model::ScaleTarget>,
2954        host_env: &RunnerEnv,
2955    ) -> Vec<(&'static str, std::ffi::OsString)> {
2956        match (&self.dependency_caches, target) {
2957            (Some(caches), Some(target)) => {
2958                caches.for_launch(target, attempt.runtime_path(), host_env)
2959            }
2960            _ => Vec::new(),
2961        }
2962    }
2963
2964    /// Starts `Runner.Listener` for `attempt`, with the dependency caches of
2965    /// `cache_target` when there is one.
2966    fn spawn_listener(
2967        &self,
2968        attempt: &RunnerAttempt,
2969        config: &EncodedJitConfig,
2970        cache_target: Option<&runner_manager_domain::model::ScaleTarget>,
2971    ) -> Result<u32, ProcessStartFailure> {
2972        let pool = self
2973            .runner
2974            .pool()
2975            .map_err(ProcessStartFailure::before_spawn)?;
2976        let account = self
2977            .account_for(attempt)
2978            .map_err(ProcessStartFailure::before_spawn)?;
2979        let handoff = RestrictiveHandoff::create(
2980            attempt.runtime_path(),
2981            SecretString::from(config.expose().to_owned()),
2982        )
2983        .map_err(|_| ProcessStartFailure::before_spawn(FailureReason::ProcessStartFailed))?;
2984        #[cfg(windows)]
2985        let program = attempt
2986            .runtime_path()
2987            .join("bin")
2988            .join("Runner.Listener.exe");
2989        #[cfg(not(windows))]
2990        let program = attempt.runtime_path().join("bin").join("Runner.Listener");
2991        // Checked after the handoff exists on purpose: the error path below is
2992        // a real post-handoff launch failure, and unwinding must delete it.
2993        if !program.is_file() {
2994            return Err(ProcessStartFailure::before_spawn(
2995                FailureReason::ProcessStartFailed,
2996            ));
2997        }
2998        let host_env = self.host_env().map_err(ProcessStartFailure::before_spawn)?;
2999        let caches = self.cache_env(attempt, cache_target, &host_env);
3000        let cache_names: Vec<&'static str> = caches.iter().map(|(name, _)| *name).collect();
3001        #[cfg(test)]
3002        let spec = if self
3003            .use_long_lived_test_listener
3004            .load(std::sync::atomic::Ordering::SeqCst)
3005        {
3006            SpawnSpec::new(program)
3007                .args([
3008                    "--ignored",
3009                    "--exact",
3010                    "lifecycle::tests::long_lived_native_listener_helper",
3011                    "--nocapture",
3012                ])
3013                .env(
3014                    "RUNNER_MANAGER_TEST_LISTENER_READY",
3015                    attempt.runtime_path().join(TEST_LISTENER_READY),
3016                )
3017                .working_dir(attempt.runtime_path())
3018        } else {
3019            runner_listener_spec(program, attempt.runtime_path(), caches, &host_env)
3020        };
3021        #[cfg(not(test))]
3022        let spec = runner_listener_spec(program, attempt.runtime_path(), caches, &host_env);
3023        let spec = match account.zip(pool) {
3024            Some((account, pool)) => {
3025                hand_over(account, pool, attempt.runtime_path(), &spec, &cache_names).map_err(
3026                    |error| {
3027                        tracing::warn!(attempt_id = %attempt.id, event = "runner_hand_over_failed", %error, "the attempt could not be handed to the runner account");
3028                        ProcessStartFailure::before_spawn(FailureReason::ProcessStartFailed)
3029                    },
3030                )?;
3031                spec.run_as(account)
3032            }
3033            None => spec,
3034        };
3035        let (spec, docker_proxy) = if let Some(cleanup) = &self.docker_cleanup {
3036            let repository = cache_target.map(ToString::to_string).unwrap_or_default();
3037            let listen =
3038                account.map(|account| (attempt.runtime_path().join(DOCKER_SOCKET_DIR), account));
3039            // What the job may bind-mount into its containers: its attempt and
3040            // the cache directories it was given.
3041            let own_directories: Vec<PathBuf> =
3042                std::iter::once(attempt.runtime_path().to_path_buf())
3043                    .chain(cache_names.iter().filter_map(|name| {
3044                        spec.configured_env(name)
3045                            .map(PathBuf::from)
3046                            .filter(|path| path.is_absolute())
3047                    }))
3048                    .collect();
3049            match cleanup.launch_environment(
3050                attempt.id,
3051                &repository,
3052                &host_env,
3053                listen,
3054                &own_directories,
3055            ) {
3056                Ok(Some(endpoint)) => (
3057                    spec.env("DOCKER_HOST", endpoint),
3058                    Some(cleanup.root.join(attempt.id.to_string())),
3059                ),
3060                Ok(None) => (spec, None),
3061                Err(error) => {
3062                    tracing::warn!(attempt_id = %attempt.id, event = "docker_accounting_unavailable", reason = ?error.kind(), "native Docker accounting could not start");
3063                    if crate::docker_cleanup::Config::load(&cleanup.config).map_or(true, |config| {
3064                        config.mode == crate::docker_cleanup::Mode::Apply
3065                    }) {
3066                        return Err(ProcessStartFailure::before_spawn(
3067                            FailureReason::ProcessStartFailed,
3068                        ));
3069                    }
3070                    (spec, None)
3071                }
3072            }
3073        } else {
3074            (spec, None)
3075        };
3076        let child = spec
3077            .spawn_runner_with_handoff(&handoff)
3078            .map_err(|_| ProcessStartFailure::before_spawn(FailureReason::ProcessStartFailed))?;
3079        // The payload is gone before any state saying "starting" is persisted.
3080        #[cfg(test)]
3081        if self.faults_at(PostSpawnBoundary::HandoffDelete) {
3082            drop(handoff);
3083            return Err(self.abort_spawned_child(child, attempt, false));
3084        }
3085        if handoff.delete().is_err() {
3086            return Err(self.abort_spawned_child(child, attempt, false));
3087        }
3088        #[cfg(test)]
3089        if self.faults_at(PostSpawnBoundary::IdentitySerialize) {
3090            return Err(self.abort_spawned_child(child, attempt, false));
3091        }
3092        let identity = match serde_json::to_vec(child.identity()) {
3093            Ok(identity) => identity,
3094            Err(_) => {
3095                return Err(self.abort_spawned_child(child, attempt, false));
3096            }
3097        };
3098        if self.persist_identity(attempt, &identity).is_err() {
3099            return Err(self.abort_spawned_child(child, attempt, true));
3100        }
3101        if let Some(directory) = docker_proxy
3102            && crate::docker_cleanup::write_json(&directory.join("runner.json"), child.identity())
3103                .is_err()
3104        {
3105            return Err(self.abort_spawned_child(child, attempt, true));
3106        }
3107        let pid = child.pid();
3108        #[cfg(test)]
3109        if self.faults_at(PostSpawnBoundary::ChildMapInsert) {
3110            return Err(self.abort_spawned_child(child, attempt, true));
3111        }
3112        let mut children = self
3113            .children
3114            .lock()
3115            .unwrap_or_else(std::sync::PoisonError::into_inner);
3116        children.insert(attempt.id, child);
3117        Ok(pid)
3118    }
3119
3120    /// The host's `runner.env`, read now.
3121    ///
3122    /// A file that cannot be read or parsed fails the launch rather than
3123    /// starting a runner without the environment the operator configured.
3124    /// [`ExecutionProvider::prepare`] asks first, before a JIT runner is
3125    /// registered, so a broken file costs no registration. The reason names
3126    /// the file and the line or variable, never a value.
3127    fn host_env(&self) -> Result<RunnerEnv, FailureReason> {
3128        let Some(path) = &self.runner_env_file else {
3129            return Ok(RunnerEnv::default());
3130        };
3131        RunnerEnv::load(path).map_err(|error| {
3132            tracing::warn!(
3133                reason = "runner_env_file_invalid",
3134                "runner.env could not be applied, so no runner starts until it is fixed: {error}"
3135            );
3136            FailureReason::Other(format!(
3137                "runner.env cannot be applied ({error}); fix it with `runner-manager host env`"
3138            ))
3139        })
3140    }
3141
3142    #[cfg(test)]
3143    fn fail_post_spawn_at(&self, boundary: PostSpawnBoundary) {
3144        self.post_spawn_faults.lock().unwrap().push_back(boundary);
3145    }
3146
3147    #[cfg(test)]
3148    fn faults_at(&self, boundary: PostSpawnBoundary) -> bool {
3149        let mut faults = self.post_spawn_faults.lock().unwrap();
3150        if faults.front() == Some(&boundary) {
3151            faults.pop_front();
3152            true
3153        } else {
3154            false
3155        }
3156    }
3157
3158    #[cfg(test)]
3159    fn fail_post_spawn_stops(&self, count: usize) {
3160        self.post_spawn_stop_failures
3161            .fetch_add(count, std::sync::atomic::Ordering::SeqCst);
3162    }
3163
3164    #[cfg(test)]
3165    fn fail_next_post_spawn_stop(&self) {
3166        self.fail_post_spawn_stops(1);
3167    }
3168
3169    #[cfg(test)]
3170    fn use_long_lived_test_listener(&self) {
3171        self.use_long_lived_test_listener
3172            .store(true, std::sync::atomic::Ordering::SeqCst);
3173    }
3174
3175    fn stop_spawned_child(&self, child: &mut ChildProcess) -> Result<(), FailureReason> {
3176        #[cfg(test)]
3177        if self
3178            .post_spawn_stop_failures
3179            .fetch_update(
3180                std::sync::atomic::Ordering::SeqCst,
3181                std::sync::atomic::Ordering::SeqCst,
3182                |left| if left > 0 { Some(left - 1) } else { None },
3183            )
3184            .is_ok()
3185        {
3186            return Err(FailureReason::Other("injected runner stop failure".into()));
3187        }
3188        child
3189            .stop(Duration::from_secs(1))
3190            .map(|_| ())
3191            .map_err(|_| FailureReason::Other("spawned runner process could not be stopped".into()))
3192    }
3193
3194    fn abort_spawned_child(
3195        &self,
3196        mut child: ChildProcess,
3197        attempt: &RunnerAttempt,
3198        remove_identity: bool,
3199    ) -> ProcessStartFailure {
3200        let mut reaped = self.stop_spawned_child(&mut child).is_ok();
3201        if reaped {
3202            if remove_identity {
3203                Self::remove_identity_files(attempt);
3204            }
3205        } else {
3206            // A failed stop is not a failed attempt yet.  Persist enough truth
3207            // for crash recovery, and retain the owned child when possible.
3208            let identity_durable =
3209                serde_json::to_vec(child.identity())
3210                    .ok()
3211                    .is_some_and(|identity| {
3212                        self.persist_identity(attempt, &identity).is_ok()
3213                            || self.persist_fallback_identity(attempt, &identity).is_ok()
3214                    });
3215            if !identity_durable {
3216                // Returning a live PID as though recovery were complete would
3217                // make the next boot trust a recyclable PID. Reaping is bounded;
3218                // if it cannot finish, the durable `starting` journal entry is
3219                // deliberately unresolved on restart and blocks new launches.
3220                for _ in 1..MAX_POST_SPAWN_STOP_ATTEMPTS {
3221                    if self.stop_spawned_child(&mut child).is_ok() {
3222                        reaped = true;
3223                        break;
3224                    }
3225                }
3226                if !reaped {
3227                    // The attempt journal will durably record `starting` and
3228                    // its PID. Recovery treats a missing full identity as
3229                    // unresolved and starts nothing, so bounded stop failure
3230                    // cannot turn into either a hang or a duplicate runner.
3231                    let pid = child.pid();
3232                    let marker = write_durable_file(
3233                        &Self::unresolved_process_path(attempt),
3234                        pid.to_string().as_bytes(),
3235                    );
3236                    self.children
3237                        .lock()
3238                        .unwrap_or_else(std::sync::PoisonError::into_inner)
3239                        .insert(attempt.id, child);
3240                    let reason = if marker.is_ok() {
3241                        FailureReason::Other(
3242                            "spawn cleanup exhausted its bounded stop attempts; the live process remains under durable unresolved supervision"
3243                                .into(),
3244                        )
3245                    } else {
3246                        FailureReason::Other(
3247                            "spawn cleanup exhausted its bounded stop attempts and the unresolved-process marker could not be journalled"
3248                                .into(),
3249                        )
3250                    };
3251                    return ProcessStartFailure::after_spawn_live_with_reason(pid, reason);
3252                }
3253            } else {
3254                let pid = child.pid();
3255                self.children
3256                    .lock()
3257                    .unwrap_or_else(std::sync::PoisonError::into_inner)
3258                    .insert(attempt.id, child);
3259                return ProcessStartFailure::after_spawn_live(pid);
3260            }
3261        }
3262        #[cfg(test)]
3263        if reaped {
3264            self.post_spawn_reaps
3265                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
3266        }
3267        #[cfg(not(test))]
3268        let _ = reaped;
3269        ProcessStartFailure::after_spawn_stopped()
3270    }
3271
3272    fn identity_path(attempt: &RunnerAttempt) -> PathBuf {
3273        attempt.runtime_path().join(IDENTITY_FILE)
3274    }
3275
3276    fn fallback_identity_path(attempt: &RunnerAttempt) -> PathBuf {
3277        attempt.runtime_path().join(FALLBACK_IDENTITY_FILE)
3278    }
3279
3280    fn unresolved_process_path(attempt: &RunnerAttempt) -> PathBuf {
3281        attempt.runtime_path().join(UNRESOLVED_PROCESS_FILE)
3282    }
3283
3284    fn remove_identity_files(attempt: &RunnerAttempt) {
3285        let _ = fs::remove_file(Self::identity_path(attempt));
3286        let _ = fs::remove_file(Self::fallback_identity_path(attempt));
3287        let _ = fs::remove_file(Self::unresolved_process_path(attempt));
3288    }
3289
3290    fn persist_identity(&self, attempt: &RunnerAttempt, bytes: &[u8]) -> std::io::Result<()> {
3291        self.persist_identity_at(&Self::identity_path(attempt), bytes)
3292    }
3293
3294    fn persist_fallback_identity(
3295        &self,
3296        attempt: &RunnerAttempt,
3297        bytes: &[u8],
3298    ) -> std::io::Result<()> {
3299        self.persist_identity_at(&Self::fallback_identity_path(attempt), bytes)
3300    }
3301
3302    fn persist_identity_at(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
3303        #[cfg(test)]
3304        if self.faults_at(PostSpawnBoundary::IdentityWrite) {
3305            return Err(std::io::Error::other("injected identity write failure"));
3306        }
3307        write_durable_file(path, bytes)
3308    }
3309
3310    fn intent_path(attempt: &RunnerAttempt) -> PathBuf {
3311        attempt.runtime_path().join(TERMINATE_INTENT_FILE)
3312    }
3313
3314    /// The recorded runner process, `None` when none was recorded.
3315    ///
3316    /// A recorded process that is alive but not the runner account's -- a
3317    /// forged record, or a PID the kernel has since given to another account
3318    /// -- is reported as [`RecordedProcess::Foreign`], which every caller
3319    /// treats as a runner that is gone. It is not `None`: an attempt that has
3320    /// a PID and no record refuses recovery, and an exited runner whose PID was
3321    /// reused must not be held up that way.
3322    fn read_identity(&self, attempt: &RunnerAttempt) -> Result<RecordedProcess, FailureReason> {
3323        let identity = match Self::read_identity_at(&Self::identity_path(attempt))? {
3324            Some(identity) => Some(identity),
3325            None => Self::read_identity_at(&Self::fallback_identity_path(attempt))?,
3326        };
3327        Ok(match identity {
3328            None => RecordedProcess::None,
3329            Some(identity) if self.runs_as_runner(attempt, &identity) => {
3330                RecordedProcess::Runner(identity)
3331            }
3332            Some(_) => RecordedProcess::Foreign,
3333        })
3334    }
3335
3336    /// Whether the process `identity` names belongs to the runner account.
3337    ///
3338    /// The identity files live in the attempt directory, which the runner's
3339    /// account owns when it is not this process's, so a job could rewrite one
3340    /// to name any process on the host for this process to signal. A recorded
3341    /// process the account does not run is therefore not one this daemon
3342    /// started, and is treated as gone. A process that has exited passes, and
3343    /// the identity's own start-time check then reports it gone.
3344    fn runs_as_runner(&self, attempt: &RunnerAttempt, identity: &ProcessIdentity) -> bool {
3345        let RunnerIdentity::Pool(pool) = &self.runner else {
3346            return true;
3347        };
3348        // The attempt's own account, read from its directory, since every
3349        // runner on the host has one: a record naming another runner's process
3350        // is as foreign as one naming any other process.
3351        let Some(account) = owner_in_pool(pool, attempt.runtime_path()) else {
3352            return false;
3353        };
3354        #[cfg(unix)]
3355        {
3356            use std::os::unix::fs::MetadataExt as _;
3357            match fs::metadata(format!("/proc/{}", identity.pid())) {
3358                Ok(metadata) => metadata.uid() == account.uid,
3359                Err(error) => error.kind() == std::io::ErrorKind::NotFound,
3360            }
3361        }
3362        #[cfg(not(unix))]
3363        {
3364            let _ = (account, identity);
3365            true
3366        }
3367    }
3368
3369    /// The pool account `attempt`'s runner runs as, assigned now when it has
3370    /// none yet; `None` when runners run as the daemon.
3371    ///
3372    /// An attempt directory a pool account already owns -- a retried launch, a
3373    /// persistent slot, an attempt that outlived a restart -- keeps that
3374    /// account. A new one gets the first account no live attempt holds: none
3375    /// in this process's assignments, and none owning a directory beside it
3376    /// under the runner root, which covers attempts that outlived a restart.
3377    /// Two runners never share an account, so a launch with every account
3378    /// busy fails rather than weaken that. A pool with only the legacy
3379    /// single account, from before the pool existed, shares it as before.
3380    fn account_for(&self, attempt: &RunnerAttempt) -> Result<Option<RunnerAccount>, FailureReason> {
3381        let Some(pool) = self.runner.pool()? else {
3382            return Ok(None);
3383        };
3384        let mut assignments = self
3385            .assignments
3386            .lock()
3387            .unwrap_or_else(std::sync::PoisonError::into_inner);
3388        if let Some(account) = assignments.get(&attempt.id) {
3389            return Ok(Some(*account));
3390        }
3391        let runtime = attempt.runtime_path();
3392        let held = assignments.values().map(|account| account.uid).collect();
3393        // The accounts owning the other attempts' directories, including ones
3394        // a previous daemon left behind.
3395        let busy = || {
3396            runtime
3397                .parent()
3398                .and_then(|root| fs::read_dir(root).ok())
3399                .into_iter()
3400                .flatten()
3401                .flatten()
3402                .filter_map(|entry| owner_in_pool(pool, &entry.path()))
3403                .map(|account| account.uid)
3404                .collect()
3405        };
3406        let account = pool
3407            .allocate(owner_in_pool(pool, runtime), &held, busy)
3408            .ok_or_else(|| {
3409                FailureReason::Other(format!(
3410                    "every one of the {} runner accounts is in use by another attempt, and no \
3411                     two runners share one",
3412                    pool.accounts.len()
3413                ))
3414            })?;
3415        assignments.insert(attempt.id, account);
3416        Ok(Some(account))
3417    }
3418
3419    fn read_identity_at(path: &Path) -> Result<Option<ProcessIdentity>, FailureReason> {
3420        match fs::read(path) {
3421            Ok(bytes) => serde_json::from_slice(&bytes)
3422                .map(Some)
3423                .map_err(|_| FailureReason::Other("process identity journal is unreadable".into())),
3424            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
3425            Err(_) => Err(FailureReason::Other(
3426                "process identity journal could not be read".into(),
3427            )),
3428        }
3429    }
3430}
3431
3432impl ProcessSupervisor for NativeProcesses {
3433    fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
3434        if !attempt.execution().is_native() {
3435            return Err(FailureReason::Other(
3436                "execution provider unavailable".into(),
3437            ));
3438        }
3439        // Existing native reconciliation has already settled process termination
3440        // before this boundary. Capture before ephemeral workspace deletion.
3441        if let Some(caches) = &self.dependency_caches {
3442            caches.capture_git(attempt.runtime_path());
3443        }
3444        // The account is free for another attempt once its runner is gone; the
3445        // directory it still owns until cleanup keeps it busy until then.
3446        self.assignments
3447            .lock()
3448            .unwrap_or_else(std::sync::PoisonError::into_inner)
3449            .remove(&attempt.id);
3450        Ok(ProviderDestroy::Destroyed)
3451    }
3452
3453    fn prepare(
3454        &self,
3455        attempt: &RunnerAttempt,
3456        policy: &ScalePolicy,
3457    ) -> Result<PreparedEnvironment, FailureReason> {
3458        if !attempt.execution().is_native() || !policy.execution_policy().is_native() {
3459            return Err(FailureReason::Other(
3460                "execution provider unavailable".into(),
3461            ));
3462        }
3463        self.host_env()?;
3464        // The materialized attempt is handed to the runner's account here, off
3465        // the async thread and before the JIT registration: it is the one walk
3466        // over the whole package copy, and the launch repeats only the cheap
3467        // part of it for what it creates afterwards.
3468        if let Some(account) = self.account_for(attempt)? {
3469            hand_over_attempt(account, attempt.runtime_path()).map_err(|error| {
3470                tracing::warn!(attempt_id = %attempt.id, event = "runner_hand_over_failed", %error, "the attempt could not be handed to the runner account");
3471                FailureReason::ProcessStartFailed
3472            })?;
3473        }
3474        // Asked here, before the JIT registration, for the reason `host_env`
3475        // is: a broken file must cost no registration.
3476        let cache_target = match &self.dependency_caches {
3477            Some(caches) => {
3478                caches.config()?;
3479                Some(policy.target.clone())
3480            }
3481            None => None,
3482        };
3483        Ok(PreparedEnvironment {
3484            attempt: attempt.id,
3485            identity: None,
3486            cache_target,
3487        })
3488    }
3489
3490    fn start(
3491        &self,
3492        prepared: PreparedEnvironment,
3493        attempt: &RunnerAttempt,
3494        handoff: OneTimeJitHandoff<'_>,
3495    ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
3496        if prepared.attempt != attempt.id || !attempt.execution().is_native() {
3497            return Err(ProcessStartFailure::before_spawn(FailureReason::Other(
3498                "execution provider identity mismatch".into(),
3499            )));
3500        }
3501        self.spawn_listener(attempt, handoff.consume(), prepared.cache_target.as_ref())
3502            .map(EnvironmentIdentity::NativeProcess)
3503    }
3504
3505    fn spawn(
3506        &self,
3507        attempt: &RunnerAttempt,
3508        config: &EncodedJitConfig,
3509    ) -> Result<u32, ProcessStartFailure> {
3510        self.spawn_listener(attempt, config, None)
3511    }
3512
3513    fn is_alive(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
3514        let mut children = self
3515            .children
3516            .lock()
3517            .unwrap_or_else(std::sync::PoisonError::into_inner);
3518        if let Some(child) = children.get_mut(&attempt.id) {
3519            return match child
3520                .try_exit_status()
3521                .map_err(|_| FailureReason::Other("runner process could not be observed".into()))?
3522            {
3523                None => Ok(true),
3524                Some(status) => {
3525                    if let Ok(mut exits) = self.successful_exits.lock() {
3526                        exits.insert(attempt.id, status.success());
3527                    }
3528                    Ok(false)
3529                }
3530            };
3531        }
3532        let identity = match self.read_identity(attempt)? {
3533            RecordedProcess::Runner(identity) => identity,
3534            RecordedProcess::Foreign => return Ok(false),
3535            RecordedProcess::None => {
3536                if attempt.process_id().is_some()
3537                    || Self::unresolved_process_path(attempt).is_file()
3538                {
3539                    return Err(FailureReason::Other(
3540                        "runner process identity is missing; refusing recovery until the process is resolved"
3541                            .into(),
3542                    ));
3543                }
3544                return Ok(false);
3545            }
3546        };
3547        match identity.recheck() {
3548            Ok(Adoption::Live) => Ok(true),
3549            Ok(Adoption::Gone | Adoption::PidRecycled { .. }) => Ok(false),
3550            Err(_) => Ok(false),
3551        }
3552    }
3553
3554    fn recovered_pid(&self, attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
3555        Ok(self
3556            .read_identity(attempt)?
3557            .runner()
3558            .map(|identity| identity.pid()))
3559    }
3560
3561    fn completed_successfully(&self, attempt: &RunnerAttempt) -> bool {
3562        self.successful_exits
3563            .lock()
3564            .ok()
3565            .and_then(|exits| exits.get(&attempt.id).copied())
3566            .unwrap_or(false)
3567    }
3568
3569    fn record_terminate_intent(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
3570        let path = Self::intent_path(attempt);
3571        write_durable_file(&path, b"registration-timeout\n")
3572            .map_err(|_| FailureReason::Other("terminate intent could not be journalled".into()))
3573    }
3574
3575    fn has_terminate_intent(&self, attempt: &RunnerAttempt) -> bool {
3576        Self::intent_path(attempt).is_file()
3577    }
3578
3579    fn terminate(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
3580        let mut children = self
3581            .children
3582            .lock()
3583            .unwrap_or_else(std::sync::PoisonError::into_inner);
3584        if let Some(child) = children.get_mut(&attempt.id) {
3585            child
3586                .stop(Duration::from_secs(10))
3587                .map_err(|_| FailureReason::Other("runner process could not be stopped".into()))?;
3588            return Ok(());
3589        }
3590        let Some(identity) = self.read_identity(attempt)?.runner() else {
3591            return Ok(());
3592        };
3593        match identity
3594            .terminate(Duration::from_secs(10))
3595            .map_err(|_| FailureReason::Other("runner process could not be stopped".into()))?
3596        {
3597            Termination::Terminated | Termination::AlreadyGone => Ok(()),
3598            Termination::RefusedPidRecycled { .. } => Err(FailureReason::Other(
3599                "runner PID was recycled; refusing to signal it".into(),
3600            )),
3601        }
3602    }
3603}
3604
3605pub struct LifecyclePorts {
3606    pub store: Arc<dyn Store>,
3607    pub github: Arc<dyn LifecycleGithub>,
3608    pub packages: Arc<dyn RuntimePackages>,
3609    pub processes: Arc<dyn ExecutionProvider>,
3610    pub clock: Arc<dyn Clock>,
3611    pub demand: Arc<dyn DemandPersistence>,
3612    pub delay: Arc<dyn RetryDelay>,
3613    pub events: Arc<dyn AttemptEventSink>,
3614    pub reconcile_events: Arc<dyn EventSink>,
3615}
3616
3617impl fmt::Debug for LifecyclePorts {
3618    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3619        f.debug_struct("LifecyclePorts")
3620            .field("store", &self.store)
3621            .field("github", &self.github)
3622            .field("packages", &self.packages)
3623            .field("processes", &self.processes)
3624            .finish_non_exhaustive()
3625    }
3626}
3627
3628#[derive(Debug, thiserror::Error)]
3629pub enum LifecycleError {
3630    #[error("attempt journal operation failed")]
3631    Journal,
3632    #[error("attempt {0} is not in the journal")]
3633    Missing(AttemptId),
3634    #[error("attempt lifecycle transition was refused")]
3635    Transition,
3636    #[error("startup recovery has not completed")]
3637    RecoveryIncomplete,
3638    /// A persistent slot could not be proven safe to scrub, so the attempt keeps
3639    /// its uncleaned state and, with it, its slot lease.
3640    ///
3641    /// Separate from [`Self::Failed`] because the two demand opposite handling
3642    /// from the same call sites. A failure aborts the pass; a quarantine must
3643    /// not, or one stuck slot would stop the host launching anything at all —
3644    /// which is precisely what `04-security-recovery.md` rules out when it says
3645    /// such an attempt "does not count as active host capacity" and "recovery
3646    /// retries the same cleanup".
3647    #[error("the persistent slot was not cleaned: {detail}")]
3648    SlotQuarantined {
3649        /// The closed-vocabulary event field; never operator or workflow text.
3650        class: &'static str,
3651        detail: String,
3652    },
3653    /// An ephemeral runtime is still held by a late process. Its journal state
3654    /// remains unchanged and ordinary supervision retries it, but startup
3655    /// recovery must not take unrelated repositories offline because of it.
3656    #[error("{}", FailureReason::WorkspaceCleanupDeferred)]
3657    WorkspaceCleanupDeferred,
3658    #[error("isolated environment cleanup is deferred")]
3659    EnvironmentCleanupDeferred,
3660    #[error("runner lifecycle failed: {0}")]
3661    Failed(FailureReason),
3662}
3663
3664impl LifecycleError {
3665    fn reason(&self) -> FailureReason {
3666        match self {
3667            Self::Failed(reason) => reason.clone(),
3668            Self::RecoveryIncomplete => FailureReason::Other("startup recovery incomplete".into()),
3669            Self::Journal => FailureReason::Other("attempt journal operation failed".into()),
3670            Self::Missing(_) => FailureReason::Other("attempt disappeared from the journal".into()),
3671            Self::Transition => FailureReason::Other("attempt transition was refused".into()),
3672            // Rendered through `Display` rather than a second copy of the same
3673            // sentence, so the two cannot drift apart.
3674            Self::SlotQuarantined { .. } => FailureReason::Other(self.to_string()),
3675            Self::WorkspaceCleanupDeferred => FailureReason::WorkspaceCleanupDeferred,
3676            Self::EnvironmentCleanupDeferred => FailureReason::Other(self.to_string()),
3677        }
3678    }
3679}
3680
3681/// Production implementation of e1's launcher port.
3682#[derive(Debug)]
3683pub struct LifecycleLauncher {
3684    host_id: HostId,
3685    app_paths: runner_manager_platform::paths::AppPaths,
3686    diagnostics_root: PathBuf,
3687    runner_group_id: u64,
3688    timeouts: RecoveryTimeouts,
3689    retry: RetryPolicy,
3690    cancel: CancelToken,
3691    ports: LifecyclePorts,
3692    recovery_complete: Mutex<bool>,
3693    versions: Mutex<BTreeMap<AttemptId, RunnerVersion>>,
3694    pending_replacements: Mutex<BTreeMap<AttemptId, ReplacementIntent>>,
3695    /// Backoff for cleanups [`Self::supervise`] tolerates and retries, so a
3696    /// workspace that cannot be removed is not retried and reported on every
3697    /// pass.
3698    cleanup_retries: Mutex<CleanBackoff>,
3699    turns: Arc<tokio::sync::Semaphore>,
3700    /// The attempts reserved and not yet up or failed: their bring-up is writing
3701    /// their journal entries, and it runs while the target's pass supervises.
3702    bringing_up: Mutex<BTreeSet<AttemptId>>,
3703}
3704
3705/// Takes an attempt out of [`LifecycleLauncher::bringing_up`], however its
3706/// bring-up ends.
3707struct BroughtUpOnDrop<'a> {
3708    attempts: &'a Mutex<BTreeSet<AttemptId>>,
3709    id: AttemptId,
3710}
3711
3712impl Drop for BroughtUpOnDrop<'_> {
3713    fn drop(&mut self) {
3714        self.attempts
3715            .lock()
3716            .unwrap_or_else(std::sync::PoisonError::into_inner)
3717            .remove(&self.id);
3718    }
3719}
3720
3721/// How many runners of one host prepare at once, whatever their target: a
3722/// quarter of its cores, at least two. A runner's package is ~9,000 files, and
3723/// on a disk jobs are using, more at once only make all of them slower. Fair,
3724/// so the ones waiting go in the order they were reserved.
3725#[must_use]
3726pub fn preparation_turns() -> Arc<tokio::sync::Semaphore> {
3727    let cores = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
3728    Arc::new(tokio::sync::Semaphore::new((cores / 4).max(2)))
3729}
3730
3731#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3732enum ReconcileProgress {
3733    Reconciled,
3734    Deferred,
3735    Replacement {
3736        attempt: AttemptId,
3737        operation: &'static str,
3738    },
3739}
3740
3741impl LifecycleLauncher {
3742    #[must_use]
3743    pub fn new(
3744        host_id: HostId,
3745        app_paths: runner_manager_platform::paths::AppPaths,
3746        diagnostics_root: impl Into<PathBuf>,
3747        runner_group_id: u64,
3748        timeouts: RecoveryTimeouts,
3749        retry: RetryPolicy,
3750        ports: LifecyclePorts,
3751    ) -> Self {
3752        Self {
3753            host_id,
3754            app_paths,
3755            diagnostics_root: diagnostics_root.into(),
3756            runner_group_id,
3757            timeouts,
3758            retry,
3759            cancel: CancelToken::new(),
3760            ports,
3761            recovery_complete: Mutex::new(false),
3762            versions: Mutex::new(BTreeMap::new()),
3763            pending_replacements: Mutex::new(BTreeMap::new()),
3764            cleanup_retries: Mutex::new(CleanBackoff::default()),
3765            turns: preparation_turns(),
3766            bringing_up: Mutex::default(),
3767        }
3768    }
3769
3770    /// Takes turns to prepare with every launcher holding `turns`, rather than
3771    /// only with itself: the targets of one host.
3772    #[must_use]
3773    pub fn with_preparation_turns(mut self, turns: Arc<tokio::sync::Semaphore>) -> Self {
3774        self.turns = turns;
3775        self
3776    }
3777
3778    /// Reconcile the entire journal before allowing a launch.  Unknown-policy
3779    /// attempts are left untouched rather than being acted on without an
3780    /// ownership proof.
3781    pub async fn recover_startup(
3782        &self,
3783        policies: &[ScalePolicy],
3784    ) -> Result<Vec<ReplacementIntent>, LifecycleError> {
3785        *self
3786            .recovery_complete
3787            .lock()
3788            .map_err(|_| LifecycleError::Journal)? = false;
3789        let by_id: BTreeMap<_, _> = policies.iter().map(|policy| (policy.id, policy)).collect();
3790        let attempts = self
3791            .ports
3792            .store
3793            .attempts()
3794            .map_err(|_| LifecycleError::Journal)?;
3795        let owned = self.ports.processes.enumerate_owned(self.host_id);
3796        for identity in &owned {
3797            if let EnvironmentIdentity::Isolated { attempt: id, .. } = identity
3798                && !attempts.iter().any(|attempt| {
3799                    attempt.state() != AttemptState::Cleaned
3800                        && identity_matches_intent(self.host_id, attempt, identity)
3801                })
3802            {
3803                // Enumeration is discovery, not authority to delete.
3804                self.ports
3805                    .events
3806                    .emit(AttemptEvent::QuarantinedOwnedResource { attempt: *id });
3807                tracing::warn!(host_id = %self.host_id, attempt_id = %id, reason = "owned_isolated_resource_without_matching_journal", "provider resource quarantined for operator review");
3808            }
3809        }
3810        let mut unresolved = false;
3811        for mut attempt in attempts {
3812            let Some(policy) = by_id.get(&attempt.policy_id) else {
3813                if attempt.counts_against_capacity() {
3814                    unresolved = true;
3815                }
3816                continue;
3817            };
3818            authorize(self.host_id, policy, &attempt).map_err(|_| LifecycleError::Journal)?;
3819            let allocation_resources = owned
3820                .iter()
3821                .filter(|identity| identity_same_allocation(self.host_id, &attempt, identity))
3822                .collect::<Vec<_>>();
3823            if allocation_resources.len() > 1 {
3824                self.ports
3825                    .events
3826                    .emit(AttemptEvent::QuarantinedOwnedResource {
3827                        attempt: attempt.id,
3828                    });
3829                tracing::warn!(attempt_id = %attempt.id, reason = "duplicate_isolated_resource_ownership", "provider resources quarantined for operator review");
3830            }
3831            if attempt.state() != AttemptState::Cleaned
3832                && matches!(
3833                    attempt.execution(),
3834                    AttemptExecution::Isolated {
3835                        environment_id: None,
3836                        ..
3837                    }
3838                )
3839                && allocation_resources.len() == 1
3840                && let Some(EnvironmentIdentity::Isolated { environment_id, .. }) =
3841                    allocation_resources.first().copied()
3842            {
3843                attempt
3844                    .prepared_environment(environment_id.clone())
3845                    .map_err(|_| LifecycleError::Transition)?;
3846                if attempt.state() == AttemptState::Preparing {
3847                    attempt
3848                        .mark_prepared(self.ports.clock.now())
3849                        .map_err(|_| LifecycleError::Transition)?;
3850                }
3851                self.record(&attempt)?;
3852            }
3853            match self.reconcile_one(policy, attempt).await? {
3854                ReconcileProgress::Deferred => {
3855                    if policy.execution_policy().is_native() {
3856                        unresolved = true;
3857                    }
3858                }
3859                ReconcileProgress::Replacement { attempt, operation } => {
3860                    self.pending_replacements
3861                        .lock()
3862                        .map_err(|_| LifecycleError::Journal)?
3863                        .insert(
3864                            attempt,
3865                            ReplacementIntent {
3866                                policy: policy.id,
3867                                previous_attempt: attempt,
3868                                operation,
3869                            },
3870                        );
3871                }
3872                ReconcileProgress::Reconciled => {}
3873            }
3874        }
3875        if unresolved {
3876            return Err(LifecycleError::RecoveryIncomplete);
3877        }
3878        *self
3879            .recovery_complete
3880            .lock()
3881            .map_err(|_| LifecycleError::Journal)? = true;
3882        Ok(self
3883            .pending_replacements
3884            .lock()
3885            .map_err(|_| LifecycleError::Journal)?
3886            .values()
3887            .copied()
3888            .collect())
3889    }
3890
3891    /// Supervise all attempts of one policy during an ordinary poll.
3892    pub async fn supervise(
3893        &self,
3894        policy: &ScalePolicy,
3895    ) -> Result<Vec<ReplacementIntent>, LifecycleError> {
3896        let mut replacements = Vec::new();
3897        self.pending_replacements
3898            .lock()
3899            .map_err(|_| LifecycleError::Journal)?
3900            .retain(|_, intent| {
3901                if intent.policy == policy.id {
3902                    replacements.push(*intent);
3903                    false
3904                } else {
3905                    true
3906                }
3907            });
3908        // Read under the set a reservation joins before it is journalled, so
3909        // every attempt read here that a bring-up may write to is known to be
3910        // one: it has no process yet and no registration, which this pass would
3911        // read as a runner gone, and what was read of it is out of date at once.
3912        let attempts = {
3913            let bringing_up = self
3914                .bringing_up
3915                .lock()
3916                .map_err(|_| LifecycleError::Journal)?;
3917            let mut attempts = self
3918                .ports
3919                .store
3920                .attempts_for_policy(policy.id)
3921                .map_err(|_| LifecycleError::Journal)?;
3922            attempts.retain(|attempt| !bringing_up.contains(&attempt.id));
3923            attempts
3924        };
3925        for attempt in attempts {
3926            authorize(self.host_id, policy, &attempt).map_err(|_| LifecycleError::Journal)?;
3927            if let ReconcileProgress::Replacement { attempt, operation } =
3928                self.reconcile_one(policy, attempt).await?
3929            {
3930                replacements.push(ReplacementIntent {
3931                    policy: policy.id,
3932                    previous_attempt: attempt,
3933                    operation,
3934                });
3935            }
3936        }
3937        Ok(replacements)
3938    }
3939
3940    async fn reconcile_one(
3941        &self,
3942        policy: &ScalePolicy,
3943        mut attempt: RunnerAttempt,
3944    ) -> Result<ReconcileProgress, LifecycleError> {
3945        if !attempt.execution().is_native() {
3946            return self.reconcile_isolated_one(policy, attempt).await;
3947        }
3948        if attempt.state() == AttemptState::Cleaned {
3949            // A child of Runner.Worker can outlive the runner process briefly.
3950            // If it recreates `_work` after `clean_attempt` removed the tree but
3951            // before the journal's final write, the journal truthfully reaches
3952            // `cleaned` and the directory appears again afterwards. Cleaned
3953            // attempts used to return here forever, making that late residue
3954            // invisible. The journalled ephemeral mode is the ownership proof:
3955            // retry the whole-tree removal, while a persistent slot deliberately
3956            // remains on disk with its retained `_work`.
3957            if matches!(attempt.workspace(), AttemptWorkspace::Ephemeral)
3958                && attempt.runtime_path().exists()
3959            {
3960                // The attempt is already terminal, has released its package
3961                // lease, and consumes no capacity. A late child may still hold
3962                // one of its recreated files open on Windows, so failure here
3963                // is residue to retry, not a reason to abort startup recovery
3964                // and take every unrelated repository offline.
3965                // Retried with backoff; the event is the one report, which the
3966                // event sink logs as a warning.
3967                let now = self.ports.clock.now();
3968                if !self.cleanup_retries().is_due(attempt.id, now) {
3969                    return Ok(ReconcileProgress::Reconciled);
3970                }
3971                match self.scrub_workspace(&attempt) {
3972                    Ok(()) => self.cleanup_retries().succeeded(attempt.id),
3973                    Err(LifecycleError::WorkspaceCleanupDeferred) => {
3974                        self.cleanup_retries().failed(attempt.id, now);
3975                        self.ports
3976                            .reconcile_events
3977                            .emit(LifecycleEvent::AttemptCleanFailed {
3978                                policy: attempt.policy_id,
3979                                attempt: attempt.id,
3980                                reason: "late_ephemeral_workspace_could_not_be_removed",
3981                            });
3982                    }
3983                    Err(error) => return Err(error),
3984                }
3985            }
3986            return Ok(ReconcileProgress::Reconciled);
3987        }
3988        if attempt.is_terminal() {
3989            self.clean_or_quarantine(&mut attempt)?;
3990            return Ok(ReconcileProgress::Reconciled);
3991        }
3992        let process_alive = self
3993            .ports
3994            .processes
3995            .inspect(&attempt)
3996            .map_err(LifecycleError::Failed)?
3997            == EnvironmentState::Running;
3998        let github = self
3999            .ports
4000            .github
4001            .observe(&policy.target, attempt.id, &self.cancel)
4002            .await;
4003
4004        // If the agent died after GitHub accepted the registration but before
4005        // the non-secret runner-id sidecar landed, inventory closes the gap.
4006        // Persist it before making any state decision so a second crash moves
4007        // the boundary forward rather than repeating it.
4008        if let Some(runner_id) = github.runner_id
4009            && read_runner_id(attempt.runtime_path()).is_none()
4010        {
4011            write_runner_id(attempt.runtime_path(), runner_id)?;
4012            self.ports
4013                .events
4014                .emit(AttemptEvent::RemoteIdentityRecovered {
4015                    attempt: attempt.id,
4016                    runner_id,
4017                });
4018        }
4019
4020        // The child identity is synced before `spawn` returns.  If the agent
4021        // crashed before the following `starting` journal write, recover that
4022        // exact PID and take the legal `jit_received -> starting` edge before
4023        // applying GitHub's authoritative idle/busy observation below.
4024        if attempt.state() == AttemptState::JitReceived
4025            && process_alive
4026            && let Some(pid) = self
4027                .ports
4028                .processes
4029                .recovered_pid(&attempt)
4030                .map_err(LifecycleError::Failed)?
4031        {
4032            attempt
4033                .started(pid, self.ports.clock.now())
4034                .map_err(|_| LifecycleError::Transition)?;
4035            self.record(&attempt)?;
4036        }
4037
4038        // What the runner's own diagnostics say about whether it ran a job.
4039        // Read before anything below can conclude the attempt, because cleanup
4040        // deletes `_diag` with the rest of the runtime directory.
4041        // A one-shot child owned by this invocation exited successfully. This
4042        // concludes the *runner attempt*, never the workflow outcome; GitHub
4043        // remains authoritative for that. GitHub's inventory is a sampled
4044        // witness: a job of a few seconds starts and ends between two passes,
4045        // and an assignment GitHub shows can be one the runner never took up.
4046        // The runner's own diagnostics decide, read before cleanup deletes
4047        // them, and GitHub's sample only decides when they say nothing. Not
4048        // while GitHub is unreachable: a registration still listed there could
4049        // not be removed, and nothing visits a concluded attempt again.
4050        if !process_alive
4051            && github.status != GithubRunnerObservation::Unreachable
4052            && matches!(
4053                attempt.state(),
4054                AttemptState::Starting | AttemptState::Idle | AttemptState::Busy
4055            )
4056            && self.ports.processes.completed_successfully(&attempt)
4057        {
4058            let state = attempt.state();
4059            let outcome = match (state, JobEvidence::of(attempt.runtime_path())) {
4060                // Without a runner id there is no edge to `busy`; the recovery
4061                // path below concludes the attempt without one.
4062                (AttemptState::Starting | AttemptState::Idle, JobEvidence::RanAJob) => {
4063                    known_runner_id(&attempt, github.runner_id)
4064                        .map(|runner_id| (Some(runner_id), AttemptOutcome::CompletedJob))
4065                }
4066                (AttemptState::Busy, JobEvidence::RanAJob) => {
4067                    Some((None, AttemptOutcome::CompletedJob))
4068                }
4069                (AttemptState::Busy, JobEvidence::NoJob) => {
4070                    Some((None, AttemptOutcome::ExitedIdleWithoutWork))
4071                }
4072                (AttemptState::Busy, JobEvidence::Unknown)
4073                    if github.status == GithubRunnerObservation::NotRegistered =>
4074                {
4075                    Some((None, AttemptOutcome::CompletedJob))
4076                }
4077                _ => None,
4078            };
4079            if let Some((walk_to_busy, outcome)) = outcome {
4080                // An ephemeral runner leaves GitHub's list a little after its
4081                // process exits; one still listed is removed here rather than
4082                // left for GitHub to time out. First, so the journal's last
4083                // intermediate state and the conclusion are written back to back:
4084                // an agent that dies between them recovers from the state it was
4085                // in before this pass.
4086                if matches!(github.status, GithubRunnerObservation::Registered { .. }) {
4087                    self.deregister_runner(policy, &attempt).await;
4088                }
4089                let now = self.ports.clock.now();
4090                match (walk_to_busy, &outcome) {
4091                    (Some(runner_id), _) => {
4092                        self.transition(&mut attempt, |a| a.assigned_job(runner_id, now))?;
4093                    }
4094                    (None, AttemptOutcome::ExitedIdleWithoutWork) => {
4095                        self.transition(&mut attempt, |a| a.assignment_not_taken(now))?;
4096                    }
4097                    (None, _) => {}
4098                }
4099                self.conclude(&mut attempt, outcome)?;
4100                self.clean_or_quarantine(&mut attempt)?;
4101                return Ok(ReconcileProgress::Reconciled);
4102            }
4103        }
4104
4105        // A live runner journalled `busy` that GitHub now shows waiting, and
4106        // whose diagnostics show no job: the assignment was never taken up.
4107        // Back to `idle`, where the idle timeout applies to it again; the
4108        // `busy` arm below would otherwise adopt it for as long as it runs.
4109        if process_alive
4110            && attempt.state() == AttemptState::Busy
4111            && github.status == (GithubRunnerObservation::Registered { busy: false })
4112            && JobEvidence::of(attempt.runtime_path()) == JobEvidence::NoJob
4113        {
4114            let now = self.ports.clock.now();
4115            self.transition(&mut attempt, |a| a.assignment_not_taken(now))?;
4116            return Ok(ReconcileProgress::Reconciled);
4117        }
4118
4119        // This durable mark is more authoritative than a later observation
4120        // which cannot distinguish an agent kill from a crash.
4121        if self.ports.processes.has_terminate_intent(&attempt) && !process_alive {
4122            self.deregister_runner(policy, &attempt).await;
4123            self.conclude(
4124                &mut attempt,
4125                AttemptOutcome::failed(FailureReason::TerminatedAfterRegistrationTimeout),
4126            )?;
4127            self.clean_or_quarantine(&mut attempt)?;
4128            return Ok(ReconcileProgress::Replacement {
4129                attempt: attempt.id,
4130                operation: "registration_timeout_replacement",
4131            });
4132        }
4133
4134        // A remote registration with no surviving process has lost its
4135        // one-shot JIT secret.  Walking it to `starting` would require inventing
4136        // a PID; retrying the same registration would require inventing the
4137        // secret.  Record the configuration as expired and let the bounded
4138        // replacement path request a fresh one only if demand remains.
4139        if matches!(
4140            attempt.state(),
4141            AttemptState::Allocated | AttemptState::JitReceived
4142        ) && !process_alive
4143            && matches!(github.status, GithubRunnerObservation::Registered { .. })
4144        {
4145            if attempt.state() == AttemptState::Allocated {
4146                attempt
4147                    .jit_received(self.ports.clock.now())
4148                    .map_err(|_| LifecycleError::Transition)?;
4149                self.record(&attempt)?;
4150            }
4151            self.deregister_runner(policy, &attempt).await;
4152            self.conclude(
4153                &mut attempt,
4154                AttemptOutcome::failed(FailureReason::JitExpired),
4155            )?;
4156            self.clean_or_quarantine(&mut attempt)?;
4157            return Ok(ReconcileProgress::Replacement {
4158                attempt: attempt.id,
4159                operation: "jit_expired_replacement",
4160            });
4161        }
4162
4163        match recovery_decision(
4164            &attempt,
4165            RecoveryObservation {
4166                process_alive,
4167                github: github.status,
4168            },
4169            self.timeouts,
4170            self.ports.clock.as_ref(),
4171        ) {
4172            RecoveryDecision::Nothing | RecoveryDecision::Wait => Ok(ReconcileProgress::Reconciled),
4173            RecoveryDecision::Defer => Ok(ReconcileProgress::Deferred),
4174            RecoveryDecision::Adopt => {
4175                self.ports.events.emit(AttemptEvent::Adopted {
4176                    attempt: attempt.id,
4177                });
4178                Ok(ReconcileProgress::Reconciled)
4179            }
4180            RecoveryDecision::Clean => {
4181                self.clean_or_quarantine(&mut attempt)?;
4182                Ok(ReconcileProgress::Reconciled)
4183            }
4184            RecoveryDecision::Observe(state) => {
4185                let runner_id = known_runner_id(&attempt, github.runner_id)
4186                    .ok_or(LifecycleError::Transition)?;
4187                match state {
4188                    AttemptState::JitReceived => attempt
4189                        .jit_received(self.ports.clock.now())
4190                        .map_err(|_| LifecycleError::Transition)?,
4191                    AttemptState::Starting => {
4192                        let pid = attempt.process_id().ok_or(LifecycleError::Transition)?;
4193                        attempt
4194                            .started(pid, self.ports.clock.now())
4195                            .map_err(|_| LifecycleError::Transition)?;
4196                    }
4197                    AttemptState::Idle => attempt
4198                        .registered_idle(runner_id, self.ports.clock.now())
4199                        .map_err(|_| LifecycleError::Transition)?,
4200                    AttemptState::Busy => attempt
4201                        .assigned_job(runner_id, self.ports.clock.now())
4202                        .map_err(|_| LifecycleError::Transition)?,
4203                    _ => return Err(LifecycleError::Transition),
4204                }
4205                self.record(&attempt)?;
4206                Ok(ReconcileProgress::Reconciled)
4207            }
4208            RecoveryDecision::Conclude(outcome) => {
4209                let replacement = replacement_operation(&outcome);
4210                // Only when GitHub still holds one. Every other conclusion here
4211                // was reached *because* the observation was `NotRegistered`, and
4212                // spending a DELETE to be told so again would put a request per
4213                // concluded attempt on a budget `rest.rs` prices to the request.
4214                if matches!(github.status, GithubRunnerObservation::Registered { .. }) {
4215                    self.deregister_runner(policy, &attempt).await;
4216                }
4217                self.conclude(&mut attempt, outcome)?;
4218                self.clean_or_quarantine(&mut attempt)?;
4219                Ok(
4220                    replacement.map_or(ReconcileProgress::Reconciled, |operation| {
4221                        ReconcileProgress::Replacement {
4222                            attempt: attempt.id,
4223                            operation,
4224                        }
4225                    }),
4226                )
4227            }
4228            RecoveryDecision::Terminate(payload) => {
4229                // The mark is synced first, and what proves the process died is
4230                // the `is_alive` re-read below -- not the outcome recorded after
4231                // it. Which outcome that is depends on why the termination was
4232                // ordered, and only the payload knows: a `starting` runner that
4233                // never registered is a failure this agent then stopped, while
4234                // an `idle` one past its timeout is flow 2.7's surplus exit and
4235                // no failure at all. Hardcoding the first reason here labelled
4236                // the second as a registration timeout and asked the allocator
4237                // for a replacement to boot.
4238                let idle_exit = payload.is_idle_exit();
4239                self.ports
4240                    .processes
4241                    .record_terminate_intent(&attempt)
4242                    .map_err(LifecycleError::Failed)?;
4243                self.ports.events.emit(AttemptEvent::TerminateIntent {
4244                    attempt: attempt.id,
4245                });
4246                if self
4247                    .ports
4248                    .processes
4249                    .stop(&attempt)
4250                    .map_err(LifecycleError::Failed)?
4251                    == ProviderStop::StillRunning
4252                {
4253                    return Ok(ReconcileProgress::Deferred);
4254                }
4255                if self
4256                    .ports
4257                    .processes
4258                    .inspect(&attempt)
4259                    .map_err(LifecycleError::Failed)?
4260                    == EnvironmentState::Running
4261                {
4262                    return Ok(ReconcileProgress::Deferred);
4263                }
4264                self.ports.events.emit(AttemptEvent::Terminated {
4265                    attempt: attempt.id,
4266                });
4267                // The registration-timeout path keeps deriving its own reason
4268                // rather than applying the payload: on the pass that reads the
4269                // journalled mark back the process is dead, and
4270                // `TerminatedAfterRegistrationTimeout` is the reason that stays
4271                // true of a dead process. See `RecoveryDecision::Terminate`.
4272                let outcome = if idle_exit {
4273                    AttemptOutcome::ExitedIdleWithoutWork
4274                } else {
4275                    AttemptOutcome::failed(FailureReason::TerminatedAfterRegistrationTimeout)
4276                };
4277                self.deregister_runner(policy, &attempt).await;
4278                self.conclude(&mut attempt, outcome)?;
4279                self.clean_or_quarantine(&mut attempt)?;
4280                // A surplus runner is not replaced. It was stopped precisely
4281                // because the work it was started for went elsewhere; asking the
4282                // allocator for another one rebuilds it every idle timeout.
4283                if idle_exit {
4284                    Ok(ReconcileProgress::Reconciled)
4285                } else {
4286                    Ok(ReconcileProgress::Replacement {
4287                        attempt: attempt.id,
4288                        operation: "registration_timeout_replacement",
4289                    })
4290                }
4291            }
4292        }
4293    }
4294
4295    async fn reconcile_isolated_one(
4296        &self,
4297        policy: &ScalePolicy,
4298        mut attempt: RunnerAttempt,
4299    ) -> Result<ReconcileProgress, LifecycleError> {
4300        if attempt.state() == AttemptState::Cleaned {
4301            return Ok(ReconcileProgress::Reconciled);
4302        }
4303        if attempt.is_terminal() {
4304            self.clean_or_quarantine(&mut attempt)?;
4305            return Ok(ReconcileProgress::Reconciled);
4306        }
4307        if matches!(
4308            attempt.state(),
4309            AttemptState::Allocated | AttemptState::Preparing | AttemptState::Prepared
4310        ) {
4311            // No JIT was requested yet. Restart can safely retire this attempt;
4312            // a matching owned resource is stopped by the durable cleanup path.
4313            self.conclude(
4314                &mut attempt,
4315                AttemptOutcome::failed(FailureReason::Other(
4316                    "isolated preparation interrupted".into(),
4317                )),
4318            )?;
4319            self.clean_or_quarantine(&mut attempt)?;
4320            return Ok(ReconcileProgress::Reconciled);
4321        }
4322        let Ok(provider_state) = self.ports.processes.recover(&attempt) else {
4323            return Ok(ReconcileProgress::Deferred);
4324        };
4325        if provider_state != EnvironmentState::Missing {
4326            let Ok(owned) = self.ports.processes.owns(&attempt) else {
4327                return Ok(ReconcileProgress::Deferred);
4328            };
4329            if !owned {
4330                self.ports
4331                    .events
4332                    .emit(AttemptEvent::QuarantinedOwnedResource {
4333                        attempt: attempt.id,
4334                    });
4335                tracing::warn!(attempt_id = %attempt.id, reason = "isolated_resource_ownership_mismatch", "provider resource quarantined for operator review");
4336                return Ok(ReconcileProgress::Deferred);
4337            }
4338        }
4339        let live = matches!(
4340            provider_state,
4341            EnvironmentState::Starting | EnvironmentState::Running
4342        );
4343        if attempt.state() == AttemptState::JitReceived && live {
4344            attempt
4345                .started_isolated(self.ports.clock.now())
4346                .map_err(|_| LifecycleError::Transition)?;
4347            self.record(&attempt)?;
4348        }
4349        let github = self
4350            .ports
4351            .github
4352            .observe(&policy.target, attempt.id, &self.cancel)
4353            .await;
4354        if let Some(runner_id) = github.runner_id
4355            && read_runner_id(attempt.runtime_path()).is_none()
4356        {
4357            write_runner_id(attempt.runtime_path(), runner_id)?;
4358        }
4359        // A successfully reaped one-shot listener is stronger evidence than a
4360        // missing registration. Recovered guests have no attached child and
4361        // still follow the conservative orphan path below.
4362        if attempt.state() == AttemptState::Busy
4363            && !live
4364            && github.status == GithubRunnerObservation::NotRegistered
4365            && self.ports.processes.completed_successfully(&attempt)
4366        {
4367            self.conclude(&mut attempt, AttemptOutcome::CompletedJob)?;
4368            self.clean_or_quarantine(&mut attempt)?;
4369            return Ok(ReconcileProgress::Reconciled);
4370        }
4371        match recovery_decision(
4372            &attempt,
4373            RecoveryObservation {
4374                process_alive: live,
4375                github: github.status,
4376            },
4377            self.timeouts,
4378            self.ports.clock.as_ref(),
4379        ) {
4380            RecoveryDecision::Nothing | RecoveryDecision::Wait => Ok(ReconcileProgress::Reconciled),
4381            RecoveryDecision::Defer => Ok(ReconcileProgress::Deferred),
4382            RecoveryDecision::Adopt => {
4383                self.ports.events.emit(AttemptEvent::Adopted {
4384                    attempt: attempt.id,
4385                });
4386                Ok(ReconcileProgress::Reconciled)
4387            }
4388            RecoveryDecision::Clean => {
4389                self.clean_or_quarantine(&mut attempt)?;
4390                Ok(ReconcileProgress::Reconciled)
4391            }
4392            RecoveryDecision::Observe(state) => {
4393                let runner_id = github
4394                    .runner_id
4395                    .or_else(|| read_runner_id(attempt.runtime_path()))
4396                    .ok_or(LifecycleError::Transition)?;
4397                match state {
4398                    AttemptState::Idle => attempt
4399                        .registered_idle(runner_id, self.ports.clock.now())
4400                        .map_err(|_| LifecycleError::Transition)?,
4401                    AttemptState::Busy => attempt
4402                        .assigned_job(runner_id, self.ports.clock.now())
4403                        .map_err(|_| LifecycleError::Transition)?,
4404                    _ => return Err(LifecycleError::Transition),
4405                }
4406                self.record(&attempt)?;
4407                Ok(ReconcileProgress::Reconciled)
4408            }
4409            RecoveryDecision::Conclude(outcome) | RecoveryDecision::Terminate(outcome) => {
4410                let replacement = replacement_operation(&outcome);
4411                if matches!(github.status, GithubRunnerObservation::Registered { .. }) {
4412                    self.deregister_runner(policy, &attempt).await;
4413                }
4414                self.conclude(&mut attempt, outcome)?;
4415                self.clean_or_quarantine(&mut attempt)?;
4416                Ok(
4417                    replacement.map_or(ReconcileProgress::Reconciled, |operation| {
4418                        ReconcileProgress::Replacement {
4419                            attempt: attempt.id,
4420                            operation,
4421                        }
4422                    }),
4423                )
4424            }
4425        }
4426    }
4427
4428    /// Takes one domain edge and journals it.
4429    fn transition(
4430        &self,
4431        attempt: &mut RunnerAttempt,
4432        edge: impl FnOnce(
4433            &mut RunnerAttempt,
4434        ) -> Result<(), runner_manager_domain::attempt::AttemptError>,
4435    ) -> Result<(), LifecycleError> {
4436        edge(attempt).map_err(|_| LifecycleError::Transition)?;
4437        self.record(attempt)
4438    }
4439
4440    fn record(&self, attempt: &RunnerAttempt) -> Result<(), LifecycleError> {
4441        self.ports
4442            .store
4443            .record_attempt(attempt)
4444            .map_err(|_| LifecycleError::Journal)?;
4445        self.ports.events.emit(AttemptEvent::State {
4446            attempt: attempt.id,
4447            state: attempt.state(),
4448        });
4449        Ok(())
4450    }
4451
4452    /// Remove the GitHub registration an attempt is about to leave behind.
4453    ///
4454    /// # Why this is not fallible, and does not block the conclusion
4455    ///
4456    /// GitHub retires an ephemeral runner itself once that runner *completes a
4457    /// job*, and for the ordinary path that is the whole story. The paths that
4458    /// reach here are the ones where it does not: a runner stopped before it
4459    /// was ever assigned work, a registration whose process died still holding
4460    /// it, a JIT configuration that expired. Nothing else deletes those, and
4461    /// before this existed nothing did — they accumulated in the target's
4462    /// runner settings, one row per attempt, for the life of the repository.
4463    ///
4464    /// It returns `()` rather than a `Result` because the alternative is worse
4465    /// in both directions. The attempt is over: its process is gone and its
4466    /// slot has to come back, so a failed delete may not abort the conclusion
4467    /// or the host leaks capacity every time GitHub is unreachable. And a
4468    /// registration that outlives this call is not lost — it is exactly the
4469    /// `Registered` + dead-process observation that
4470    /// [`AttemptOutcome::Orphaned`] already names, which a later pass can still
4471    /// see. So a failure is logged and stepped over, deliberately.
4472    async fn deregister_runner(&self, policy: &ScalePolicy, attempt: &RunnerAttempt) {
4473        let Some(runner_id) = attempt
4474            .github_runner_id()
4475            .or_else(|| read_runner_id(attempt.runtime_path()))
4476        else {
4477            return;
4478        };
4479        if self
4480            .ports
4481            .github
4482            .deregister(&policy.target, runner_id, &self.cancel)
4483            .await
4484        {
4485            self.ports.events.emit(AttemptEvent::Deregistered {
4486                attempt: attempt.id,
4487                runner_id,
4488            });
4489        } else {
4490            tracing::warn!(
4491                attempt = %attempt.id,
4492                runner_id,
4493                "the runner registration could not be removed from GitHub; it will show in the \
4494                 target's runner settings until GitHub retires it or a later pass removes it"
4495            );
4496        }
4497    }
4498
4499    fn conclude(
4500        &self,
4501        attempt: &mut RunnerAttempt,
4502        outcome: AttemptOutcome,
4503    ) -> Result<(), LifecycleError> {
4504        attempt
4505            .conclude(outcome.clone(), self.ports.clock.now())
4506            .map_err(|_| LifecycleError::Transition)?;
4507        self.record(attempt)?;
4508        self.ports.events.emit(AttemptEvent::Concluded {
4509            attempt: attempt.id,
4510            outcome: OutcomeKind::of(&outcome),
4511        });
4512        Ok(())
4513    }
4514
4515    /// Clean a concluded attempt, tolerating a quarantined persistent slot.
4516    ///
4517    /// The quarantine is reported and stepped over rather than raised, because
4518    /// raising it aborts the whole pass: on the startup path that leaves
4519    /// `recovery_complete` false and stops the host launching anything, which is
4520    /// the opposite of `04-security-recovery.md`'s "it does not count as active
4521    /// host capacity" and "recovery retries the same cleanup". The attempt keeps
4522    /// its state, so it keeps its slot lease and its directory, and the next
4523    /// pass — [`crate::reconcile::Reconciler`]'s terminal sweep on every poll,
4524    /// or the next startup — attempts exactly the same cleanup again.
4525    ///
4526    /// A persistent-slot quarantine and an ephemeral directory still held by a
4527    /// late process are tolerated. A journal failure or a package lease that
4528    /// cannot be released still propagates: those are not one workspace's
4529    /// problem.
4530    ///
4531    /// A tolerated failure is retried with per-attempt backoff: until it is
4532    /// due again the attempt is skipped silently rather than re-tried and
4533    /// re-reported on every pass.
4534    fn clean_or_quarantine(&self, attempt: &mut RunnerAttempt) -> Result<(), LifecycleError> {
4535        let now = self.ports.clock.now();
4536        if !self.cleanup_retries().is_due(attempt.id, now) {
4537            return Ok(());
4538        }
4539        let reason = match self.clean_attempt(attempt) {
4540            Err(LifecycleError::SlotQuarantined { class, .. }) => class,
4541            Err(LifecycleError::WorkspaceCleanupDeferred) => {
4542                failure_reason_kind(&FailureReason::WorkspaceCleanupDeferred)
4543            }
4544            Err(LifecycleError::EnvironmentCleanupDeferred) => {
4545                "isolated_environment_cleanup_deferred"
4546            }
4547            other => {
4548                if other.is_ok() {
4549                    self.cleanup_retries().succeeded(attempt.id);
4550                }
4551                return other;
4552            }
4553        };
4554        self.cleanup_retries().failed(attempt.id, now);
4555        self.ports
4556            .reconcile_events
4557            .emit(LifecycleEvent::AttemptCleanFailed {
4558                policy: attempt.policy_id,
4559                attempt: attempt.id,
4560                reason,
4561            });
4562        Ok(())
4563    }
4564
4565    fn cleanup_retries(&self) -> std::sync::MutexGuard<'_, CleanBackoff> {
4566        self.cleanup_retries
4567            .lock()
4568            .unwrap_or_else(std::sync::PoisonError::into_inner)
4569    }
4570
4571    fn clean_attempt(&self, attempt: &mut RunnerAttempt) -> Result<(), LifecycleError> {
4572        if !attempt.execution().is_native() {
4573            return self.clean_isolated_attempt(attempt);
4574        }
4575        match self
4576            .ports
4577            .processes
4578            .destroy(attempt)
4579            .map_err(LifecycleError::Failed)?
4580        {
4581            ProviderDestroy::Destroyed => {}
4582            ProviderDestroy::Deferred(reason) => {
4583                return Err(LifecycleError::Failed(FailureReason::Other(reason.into())));
4584            }
4585        }
4586        let outcome = attempt
4587            .outcome()
4588            .cloned()
4589            .ok_or(LifecycleError::Transition)?;
4590        self.preserve_diagnostics(attempt, &outcome)?;
4591        self.scrub_workspace(attempt)?;
4592        self.ports
4593            .packages
4594            .release(attempt.id)
4595            .map_err(LifecycleError::Failed)?;
4596        attempt
4597            .clean(self.ports.clock.now())
4598            .map_err(|_| LifecycleError::Transition)?;
4599        self.record(attempt)?;
4600        let kind = OutcomeKind::of(&outcome);
4601        self.ports.events.emit(AttemptEvent::Cleaned {
4602            attempt: attempt.id,
4603            outcome: kind,
4604        });
4605        self.ports
4606            .reconcile_events
4607            .emit(LifecycleEvent::AttemptCleaned {
4608                policy: attempt.policy_id,
4609                attempt: attempt.id,
4610                outcome: kind,
4611            });
4612        Ok(())
4613    }
4614
4615    fn clean_isolated_attempt(&self, attempt: &mut RunnerAttempt) -> Result<(), LifecycleError> {
4616        if attempt.state() != AttemptState::Destroying {
4617            attempt
4618                .begin_destroy(self.ports.clock.now())
4619                .map_err(|_| LifecycleError::Transition)?;
4620            self.record(attempt)?;
4621        }
4622        let provider = self.ports.processes.as_ref();
4623        let allocation_resources = provider.enumerate_owned(self.host_id);
4624        let matching_allocation = allocation_resources
4625            .iter()
4626            .filter(|identity| identity_same_allocation(self.host_id, attempt, identity))
4627            .collect::<Vec<_>>();
4628        if matching_allocation.len() > 1
4629            || matching_allocation
4630                .iter()
4631                .any(|identity| !identity_matches_intent(self.host_id, attempt, identity))
4632        {
4633            self.ports
4634                .events
4635                .emit(AttemptEvent::QuarantinedOwnedResource {
4636                    attempt: attempt.id,
4637                });
4638            tracing::warn!(attempt_id = %attempt.id, reason = "duplicate_or_mismatched_isolated_ownership", "provider resources quarantined for operator review");
4639            return Err(self.defer_environment_cleanup(attempt)?);
4640        }
4641        let has_identity = matches!(
4642            attempt.execution(),
4643            AttemptExecution::Isolated {
4644                environment_id: Some(_),
4645                ..
4646            }
4647        );
4648        if !has_identity
4649            && allocation_resources
4650                .iter()
4651                .any(|identity| identity_matches_intent(self.host_id, attempt, identity))
4652        {
4653            return Err(self.defer_environment_cleanup(attempt)?);
4654        }
4655        let current = if has_identity {
4656            provider.inspect(attempt)
4657        } else {
4658            // A prepare effect may have succeeded before its identity could be
4659            // journalled. Empty enumeration alone is not proof of absence.
4660            provider.recover(attempt)
4661        };
4662        let current = match current {
4663            Ok(state) => state,
4664            Err(_) => return Err(self.defer_environment_cleanup(attempt)?),
4665        };
4666        if !has_identity && current != EnvironmentState::Missing {
4667            return Err(self.defer_environment_cleanup(attempt)?);
4668        }
4669        if current != EnvironmentState::Missing {
4670            if !matches!(provider.owns(attempt), Ok(true)) {
4671                self.ports
4672                    .events
4673                    .emit(AttemptEvent::QuarantinedOwnedResource {
4674                        attempt: attempt.id,
4675                    });
4676                tracing::warn!(attempt_id = %attempt.id, reason = "isolated_resource_ownership_mismatch", "provider resource quarantined for operator review");
4677                return Err(self.defer_environment_cleanup(attempt)?);
4678            }
4679            if matches!(
4680                current,
4681                EnvironmentState::Starting | EnvironmentState::Running
4682            ) && !matches!(provider.stop(attempt), Ok(ProviderStop::Stopped))
4683            {
4684                return Err(self.defer_environment_cleanup(attempt)?);
4685            }
4686            if !matches!(provider.destroy(attempt), Ok(ProviderDestroy::Destroyed)) {
4687                return Err(self.defer_environment_cleanup(attempt)?);
4688            }
4689            if !matches!(provider.inspect(attempt), Ok(EnvironmentState::Missing)) {
4690                return Err(self.defer_environment_cleanup(attempt)?);
4691            }
4692        }
4693        if provider
4694            .enumerate_owned(self.host_id)
4695            .iter()
4696            .any(|identity| identity_same_allocation(self.host_id, attempt, identity))
4697        {
4698            return Err(self.defer_environment_cleanup(attempt)?);
4699        }
4700        let outcome = attempt
4701            .outcome()
4702            .cloned()
4703            .ok_or(LifecycleError::Transition)?;
4704        // Provider diagnostics are a closed vocabulary and bounded before they
4705        // can enter tracing, even if an adapter returns an unbounded vector.
4706        for diagnostic in provider.diagnostics(attempt).into_iter().take(32) {
4707            tracing::debug!(attempt_id = %attempt.id, provider_diagnostic = ?diagnostic, "provider lifecycle diagnostic");
4708        }
4709        self.preserve_diagnostics(attempt, &outcome)?;
4710        self.scrub_workspace(attempt)?;
4711        self.ports
4712            .packages
4713            .release(attempt.id)
4714            .map_err(LifecycleError::Failed)?;
4715        attempt
4716            .clean_isolated(self.ports.clock.now())
4717            .map_err(|_| LifecycleError::Transition)?;
4718        self.record(attempt)?;
4719        self.ports.events.emit(AttemptEvent::Cleaned {
4720            attempt: attempt.id,
4721            outcome: OutcomeKind::of(&outcome),
4722        });
4723        self.ports
4724            .reconcile_events
4725            .emit(LifecycleEvent::AttemptCleaned {
4726                policy: attempt.policy_id,
4727                attempt: attempt.id,
4728                outcome: OutcomeKind::of(&outcome),
4729            });
4730        Ok(())
4731    }
4732
4733    fn defer_environment_cleanup(
4734        &self,
4735        attempt: &mut RunnerAttempt,
4736    ) -> Result<LifecycleError, LifecycleError> {
4737        attempt
4738            .defer_cleanup(self.ports.clock.now())
4739            .map_err(|_| LifecycleError::Transition)?;
4740        self.record(attempt)?;
4741        Ok(LifecycleError::EnvironmentCleanupDeferred)
4742    }
4743
4744    /// Undo an attempt's placement by the algorithm its journalled workspace
4745    /// kind makes legal (`02-target-architecture.md`, "Cleanup and recovery").
4746    ///
4747    /// The dispatch is on the *journal*, never on what the directory looks like
4748    /// now. A slot whose `_work` was replaced by a junction is still scrubbed as
4749    /// a slot rather than removed whole, and a disposable directory that happens
4750    /// to contain a `_work` still goes whole rather than being spared: the
4751    /// workspace kind is immutable precisely so that the shape of a directory a
4752    /// workflow can write to cannot choose the algorithm applied to it.
4753    fn scrub_workspace(&self, attempt: &RunnerAttempt) -> Result<(), LifecycleError> {
4754        #[cfg(test)]
4755        {
4756            // The two contamination mutants `f1` drives the security gates with.
4757            // They sit in front of the dispatch rather than inside one arm so
4758            // that a skipped cleanup is equally observable in both modes.
4759            if matches!(
4760                std::env::var("RUNNER_MANAGER_TEST_MUTANT").as_deref(),
4761                Ok("skip_workspace_cleanup" | "reuse_job_workspace")
4762            ) {
4763                return Ok(());
4764            }
4765        }
4766        match attempt.workspace() {
4767            AttemptWorkspace::Ephemeral => match remove_runtime_tree(attempt.runtime_path()) {
4768                Ok(()) => Ok(()),
4769                Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
4770                Err(_) => Err(LifecycleError::WorkspaceCleanupDeferred),
4771            },
4772            AttemptWorkspace::PersistentSlot { slot } => self.scrub_persistent_slot(attempt, slot),
4773        }
4774    }
4775
4776    /// Retain exactly `_work` and prove everything else is gone.
4777    ///
4778    /// The seven ordered checks of `04-security-recovery.md`, "Safe path
4779    /// handling": the journalled root and slot, a surviving policy's agreement,
4780    /// lexical and canonical containment, a literal enumeration, the one-entry
4781    /// allowlist, and a verification pass before the caller releases the package
4782    /// lease and marks the attempt cleaned. Any of them may refuse, and a
4783    /// refusal removes nothing after it.
4784    ///
4785    /// A slot directory that is already gone is not a refusal. There is nothing
4786    /// to scrub and nothing to prove absent, which is the same tolerance the
4787    /// disposable arm has always had for a runtime that vanished under it.
4788    fn scrub_persistent_slot(
4789        &self,
4790        attempt: &RunnerAttempt,
4791        slot: NonZeroU16,
4792    ) -> Result<(), LifecycleError> {
4793        // The policy is read from the journal, never inferred from the
4794        // directory tree, and its absence is legal: `Reconciler` cleans every
4795        // concluded attempt whether or not its policy still exists, and this
4796        // one's root is already journalled on the attempt itself.
4797        let configured = self
4798            .ports
4799            .store
4800            .policy(attempt.policy_id)
4801            .map_err(|_| LifecycleError::Journal)?
4802            .and_then(|policy| match policy.workspace_policy() {
4803                WorkspacePolicy::Persistent { root } => Some(root.clone()),
4804                WorkspacePolicy::Ephemeral => None,
4805            });
4806        let runtime = attempt.runtime_path();
4807        self.quarantine_on_refusal(
4808            attempt,
4809            verify_journalled_slot(runtime, slot, configured.as_ref())
4810                .and_then(|()| slot_is_present(runtime))
4811                .and_then(|present| {
4812                    if present {
4813                        scrub_slot_entries(runtime).and_then(|()| verify_slot_scrubbed(runtime))
4814                    } else {
4815                        Ok(())
4816                    }
4817                }),
4818        )
4819    }
4820
4821    /// Turn a slot refusal into the error that keeps the attempt uncleaned.
4822    ///
4823    /// The warning is emitted here, at the one place a quarantine is minted, so
4824    /// that both routes out of cleanup carry it: the reconciler's terminal sweep,
4825    /// which receives the error through the launcher port, and
4826    /// [`Self::clean_or_quarantine`], which swallows it to keep the pass alive.
4827    /// Everything logged is either a path this product configured or a constant
4828    /// from this module — see [`SlotQuarantine`] for why that matters.
4829    fn quarantine_on_refusal(
4830        &self,
4831        attempt: &RunnerAttempt,
4832        outcome: Result<(), SlotQuarantine>,
4833    ) -> Result<(), LifecycleError> {
4834        let Err(quarantine) = outcome else {
4835            return Ok(());
4836        };
4837        let detail = quarantine.to_string();
4838        tracing::warn!(
4839            attempt = %attempt.id,
4840            policy = %attempt.policy_id,
4841            slot = attempt.workspace().slot_number(),
4842            refusal = quarantine.refusal.class(),
4843            "{detail}"
4844        );
4845        Err(LifecycleError::SlotQuarantined {
4846            class: quarantine.refusal.class(),
4847            detail,
4848        })
4849    }
4850
4851    fn preserve_diagnostics(
4852        &self,
4853        attempt: &RunnerAttempt,
4854        outcome: &AttemptOutcome,
4855    ) -> Result<(), LifecycleError> {
4856        fs::create_dir_all(&self.diagnostics_root).map_err(|_| {
4857            LifecycleError::Failed(FailureReason::Other(
4858                "diagnostics directory could not be created".into(),
4859            ))
4860        })?;
4861        // Intentionally constructed from typed local facts, not runner output.
4862        // Raw child output can contain workflow secrets and is never copied.
4863        let reason_kind = match outcome {
4864            AttemptOutcome::Failed { reason } => failure_reason_kind(reason),
4865            _ => "none",
4866        };
4867        let diagnostic = format!(
4868            "attempt_id={}\npolicy_id={}\noutcome={}\nreason={}\n",
4869            attempt.id,
4870            attempt.policy_id,
4871            OutcomeKind::of(outcome).as_str(),
4872            reason_kind,
4873        );
4874        fs::write(
4875            self.diagnostics_root.join(format!("{}.log", attempt.id)),
4876            diagnostic,
4877        )
4878        .map_err(|_| {
4879            LifecycleError::Failed(FailureReason::Other(
4880                "redacted diagnostics could not be preserved".into(),
4881            ))
4882        })
4883    }
4884
4885    async fn materialize_with_retry(
4886        &self,
4887        policy: &ScalePolicy,
4888        attempt: &RunnerAttempt,
4889        timings: &mut PackageTimings,
4890    ) -> Result<RunnerVersion, FailureReason> {
4891        let mut issued = 0_u32;
4892        loop {
4893            issued = issued.saturating_add(1);
4894            match self.ports.packages.materialize(attempt, timings).await {
4895                Ok(version) => return Ok(version),
4896                Err(reason)
4897                    if package_failure_is_terminal(&reason)
4898                        || issued >= self.retry.max_attempts.max(1) =>
4899                {
4900                    return Err(reason);
4901                }
4902                Err(reason) => {
4903                    if !self.ports.demand.persists(policy.id).await {
4904                        return Err(reason);
4905                    }
4906                    let delay = self.retry.delay(issued);
4907                    self.ports.events.emit(AttemptEvent::Retry {
4908                        attempt: attempt.id,
4909                        operation: "package_materialization",
4910                        delay,
4911                    });
4912                    self.ports.delay.wait(delay).await;
4913                    if !self.ports.demand.persists(policy.id).await {
4914                        return Err(reason);
4915                    }
4916                }
4917            }
4918        }
4919    }
4920
4921    async fn register_with_retry(
4922        &self,
4923        policy: &ScalePolicy,
4924        attempt: AttemptId,
4925        request: &JitRunnerRequest,
4926    ) -> Result<JitRegistration, LifecycleError> {
4927        let mut issued = 0_u32;
4928        loop {
4929            issued = issued.saturating_add(1);
4930            match self
4931                .ports
4932                .github
4933                .register(&policy.target, request, &self.cancel)
4934                .await
4935            {
4936                Ok(registration) => return Ok(registration),
4937                Err(error) if error.terminal => {
4938                    return Err(LifecycleError::Failed(error.reason));
4939                }
4940                Err(error) => {
4941                    if issued >= self.retry.max_attempts.max(1)
4942                        || !self.ports.demand.persists(policy.id).await
4943                    {
4944                        return Err(LifecycleError::Failed(error.reason));
4945                    }
4946                    let delay = error
4947                        .retry_after
4948                        .unwrap_or_else(|| self.retry.delay(issued));
4949                    self.ports.events.emit(AttemptEvent::Retry {
4950                        attempt,
4951                        operation: "jit_request",
4952                        delay,
4953                    });
4954                    self.ports.delay.wait(delay).await;
4955                    if !self.ports.demand.persists(policy.id).await {
4956                        return Err(LifecycleError::Failed(error.reason));
4957                    }
4958                }
4959            }
4960        }
4961    }
4962
4963    /// Where one attempt's files go, decided while the host allocation lock is
4964    /// held and before anything external happens.
4965    ///
4966    /// The branch is on the *repository's configured* workspace policy, so an
4967    /// organization policy and an ephemeral repository never reach slot
4968    /// selection at all: a persistent policy is unrepresentable for an
4969    /// organization target (D7, refused by `WorkspacePolicy::permitted_for` in
4970    /// both the constructor and the loader), and an ephemeral repository takes
4971    /// the disposable arm that existed before slots did.
4972    fn allocate_workspace(
4973        &self,
4974        policy: &ScalePolicy,
4975        id: AttemptId,
4976    ) -> Result<Placement, LifecycleError> {
4977        let placement = match policy.workspace_policy() {
4978            // Precedence (`02-target-architecture.md`): the repository's
4979            // persistent root is selected *before* the host root, which is why
4980            // this arm is first and why it never makes resolving the host
4981            // default a precondition of its own success.
4982            WorkspacePolicy::Persistent { root } => self.allocate_persistent_slot(policy, root),
4983            WorkspacePolicy::Ephemeral => self.allocate_disposable(policy, id),
4984        };
4985        // Here rather than inside the two arms, so that every path a root can
4986        // accept clears the record and none can be forgotten.
4987        if placement.is_ok() {
4988            self.root_accepted(policy.id);
4989        }
4990        placement
4991    }
4992
4993    /// `Host.runner_root_override`, read from the journal.
4994    ///
4995    /// Separated from [`Self::effective_host_root`] so that the two failures it
4996    /// folds together stay apart: an unreadable or missing host row is a journal
4997    /// problem and is always fatal, while an unresolvable *platform default* is
4998    /// only fatal to a placement that actually needs the host root.
4999    fn configured_host_root(&self) -> Result<Option<LocalAbsolutePath>, LifecycleError> {
5000        let host = self
5001            .ports
5002            .store
5003            .host(self.host_id)
5004            .map_err(|_| LifecycleError::Journal)?
5005            .ok_or_else(|| LifecycleError::Failed(FailureReason::Other("host not found".into())))?;
5006        Ok(host.runner_root_override.clone())
5007    }
5008
5009    /// `Host.runner_root_override`, or the platform default when that resolves;
5010    /// `None` when neither does. Unlike [`Self::effective_host_root`] it records
5011    /// no refusal, for callers to whom an unresolvable root is not a failure.
5012    fn resolvable_host_root(&self) -> Result<Option<LocalAbsolutePath>, LifecycleError> {
5013        Ok(self
5014            .configured_host_root()?
5015            .or_else(|| default_runner_root(&self.app_paths).ok()))
5016    }
5017
5018    /// `Host.runner_root_override`, or the platform default standing in for it.
5019    ///
5020    /// Takes the policy because an unresolvable default is a refusal like any
5021    /// other, and the record it leaves is that policy's.
5022    fn effective_host_root(
5023        &self,
5024        policy: &ScalePolicy,
5025    ) -> Result<LocalAbsolutePath, LifecycleError> {
5026        match self.configured_host_root()? {
5027            Some(configured) => Ok(configured),
5028            None => default_runner_root(&self.app_paths).map_err(|error| {
5029                // No root resolved, so there is no path to name but the one the
5030                // platform would have produced.
5031                self.root_refused(policy.id, "the platform default runner root", error)
5032            }),
5033        }
5034    }
5035
5036    /// Turns a runner-root refusal into a failure, and leaves the sentence
5037    /// somewhere an operator can read it.
5038    ///
5039    /// Per policy, because the policies on a host do not share a fate: one with
5040    /// its own persistent root places runners while another on a withheld
5041    /// volume places none, and a host-wide record would have the first clear
5042    /// the second's on the same pass.
5043    ///
5044    /// The recording is best-effort and its failure is deliberately swallowed:
5045    /// this runs on the path that is already failing, and a host that cannot
5046    /// write a diagnostic file must still report the refusal it came to report.
5047    /// `service status` says so itself when the file is unreadable.
5048    fn root_refused(&self, policy: PolicyId, root: &str, error: RunnerRootError) -> LifecycleError {
5049        let _ = runner_manager_platform::service::record_runner_root_refusal(
5050            &self.app_paths,
5051            &policy.to_string(),
5052            self.ports.clock.now(),
5053            error.kind(),
5054            root,
5055            &error.to_string(),
5056        );
5057        root_failure(error)
5058    }
5059
5060    /// Clears that policy's record, because its root accepted a placement.
5061    ///
5062    /// Called on every successful placement rather than only after a failure:
5063    /// the daemon that recovers is often not the process that failed -- a
5064    /// self-update restarts it -- so "clear it if we wrote it" would leave a
5065    /// stale note on `service status` for as long as the host ran.
5066    fn root_accepted(&self, policy: PolicyId) {
5067        let _ = runner_manager_platform::service::clear_runner_root_refusal(
5068            &self.app_paths,
5069            &policy.to_string(),
5070        );
5071    }
5072
5073    /// D3's disposable placement: a unique child of the effective host root,
5074    /// removed whole on cleanup. `c1`'s behaviour, moved behind the branch.
5075    fn allocate_disposable(
5076        &self,
5077        policy: &ScalePolicy,
5078        id: AttemptId,
5079    ) -> Result<Placement, LifecycleError> {
5080        let effective_root = self.effective_host_root(policy)?;
5081        RootPreflight::new(&self.app_paths)
5082            .check(&RootOwner::Host, &effective_root)
5083            .map_err(|error| self.root_refused(policy.id, effective_root.as_str(), error))?;
5084        let runtime = effective_root.as_path().join({
5085            #[cfg(test)]
5086            {
5087                if std::env::var("RUNNER_MANAGER_TEST_MUTANT").as_deref()
5088                    == Ok("reuse_job_workspace")
5089                {
5090                    "mutant-shared-workspace".to_owned()
5091                } else {
5092                    workspace_name(id)
5093                }
5094            }
5095            #[cfg(not(test))]
5096            {
5097                workspace_name(id)
5098            }
5099        });
5100        fs::create_dir_all(&runtime)
5101            .map_err(|_| LifecycleError::Failed(FailureReason::ProcessStartFailed))?;
5102        Ok(Placement {
5103            runtime,
5104            workspace: AttemptWorkspace::Ephemeral,
5105        })
5106    }
5107
5108    /// D4/D5's persistent placement: the lowest free `sN` under the repository's
5109    /// configured root.
5110    ///
5111    /// Steps 1 to 6 of `02-target-architecture.md`, "Slot allocation", in order;
5112    /// step 7 is the journal write [`Self::record_allocation`] owns. All of them
5113    /// run under the host allocation lock, because [`Self::reserve_attempt`] is
5114    /// reachable only through a `LaunchRequest` and that carries the guard.
5115    ///
5116    /// **The filesystem is never consulted to decide which slots are taken**
5117    /// (invariant 6). The leases come from the journal; the directory is
5118    /// inspected only to decide whether *this* slot is safe to reuse.
5119    fn allocate_persistent_slot(
5120        &self,
5121        policy: &ScalePolicy,
5122        root: &LocalAbsolutePath,
5123    ) -> Result<Placement, LifecycleError> {
5124        // 1-4. The lowest positive slot no uncleaned attempt holds, refused
5125        // above the policy ceiling.
5126        let ceiling = policy.max_capacity().ok_or_else(|| {
5127            LifecycleError::Failed(FailureReason::Other(
5128                "a persistent workspace needs the policy's max_capacity to bound its slots"
5129                    .to_string(),
5130            ))
5131        })?;
5132        let leases = self
5133            .ports
5134            .store
5135            .slot_leases_for_policy(policy.id)
5136            .map_err(|_| LifecycleError::Journal)?;
5137        let slot = lowest_free_slot(&leases, ceiling).ok_or_else(|| {
5138            LifecycleError::Failed(FailureReason::Other(format!(
5139                "every persistent slot s1 to s{ceiling} for {} is leased by an attempt that has \
5140                 not been cleaned, so no slot is free; raise the repository's max capacity, or \
5141                 finish cleaning a concluded attempt",
5142                policy.target
5143            )))
5144        })?;
5145        let workspace = AttemptWorkspace::persistent_slot(slot);
5146        let name = workspace
5147            .slot_directory_name()
5148            .expect("a persistent allocation names its slot directory");
5149
5150        // The operational preflight, for the reasons the host root gets one: a
5151        // root that is remote, unwritable, or overlapping application data has
5152        // to fail before a directory is created rather than after. The host
5153        // root is registered only as something *not* to overlap; a host default
5154        // that cannot be resolved is a host-root problem and does not block a
5155        // repository that configured a root of its own.
5156        //
5157        // Only *that* failure is tolerated. An unreadable host row is a journal
5158        // failure and propagates, because silently continuing would drop the
5159        // overlap check entirely and accept a repository root that sits inside
5160        // the host root — the pair `RootPreflight` exists to refuse.
5161        let host_root = self.resolvable_host_root()?;
5162        let mut preflight = RootPreflight::new(&self.app_paths);
5163        if let Some(host_root) = host_root {
5164            preflight = preflight.against(RootOwner::Host, host_root);
5165        }
5166        let checked = preflight
5167            .check(&RootOwner::Repository(policy.target.to_string()), root)
5168            .map_err(|error| self.root_refused(policy.id, root.as_str(), error))?;
5169        if let Some(leaf) = checked.leaf_to_create() {
5170            fs::create_dir(leaf).map_err(|source| {
5171                LifecycleError::Failed(FailureReason::Other(format!(
5172                    "the persistent workspace root {} could not be created: {source}",
5173                    leaf.display()
5174                )))
5175            })?;
5176        }
5177
5178        // 5-6. `<root>/sN`, contained lexically by construction, then created or
5179        // validated, then contained canonically now that it resolves, and only
5180        // then accepted for reuse.
5181        let slot_path = runner_root::derive_child(root, &name)
5182            .map_err(|error| self.root_refused(policy.id, root.as_str(), error))?;
5183        create_or_validate_slot(slot_path.as_path())?;
5184        runner_root::verify_containment(root, &slot_path)
5185            .map_err(|error| self.root_refused(policy.id, root.as_str(), error))?;
5186        accept_reusable_slot(slot_path.as_path())?;
5187        Ok(Placement {
5188            runtime: slot_path.as_path().to_path_buf(),
5189            workspace,
5190        })
5191    }
5192
5193    /// The first journal write of an attempt, where a duplicate slot lease is
5194    /// still possible and has to be reported as itself.
5195    ///
5196    /// [`Self::record`] flattens every store failure into
5197    /// [`LifecycleError::Journal`], which is right for a state transition and
5198    /// wrong here: the partial unique index
5199    /// `one_uncleaned_persistent_attempt_per_slot` is the final race fence
5200    /// (`04-security-recovery.md`, "two attempts use one slot concurrently"),
5201    /// and an operator who reaches it needs to read that rather than "attempt
5202    /// journal operation failed". Nothing was written, so the caller returns
5203    /// without concluding an attempt that is not in the journal.
5204    fn record_allocation(&self, attempt: &RunnerAttempt) -> Result<(), LifecycleError> {
5205        match self.ports.store.record_attempt(attempt) {
5206            Ok(()) => {
5207                self.ports.events.emit(AttemptEvent::State {
5208                    attempt: attempt.id,
5209                    state: attempt.state(),
5210                });
5211                Ok(())
5212            }
5213            Err(error @ StoreError::SlotAlreadyLeased { .. }) => Err(LifecycleError::Failed(
5214                FailureReason::Other(error.to_string()),
5215            )),
5216            Err(_) => Err(LifecycleError::Journal),
5217        }
5218    }
5219
5220    /// Under the allocation lock: what no other launch may decide at the same
5221    /// time. The rest is [`Self::finish_attempt`], outside it.
5222    fn reserve_attempt(
5223        &self,
5224        policy: &ScalePolicy,
5225        allocation_guard: &AllocationGuard,
5226    ) -> Result<RunnerAttempt, LifecycleError> {
5227        // What a prune leaves costs only disk, and a tree it cannot remove is one
5228        // the launch's install may yet repair: not a reason to refuse the launch.
5229        if let Err(error) = self.prune_under_allocation_lock(allocation_guard) {
5230            tracing::warn!(
5231                event = "package_prune_failed",
5232                reason = failure_reason_kind(&error.reason()),
5233            );
5234        }
5235        self.allocate_attempt(policy)
5236    }
5237
5238    async fn finish_attempt(
5239        &self,
5240        policy: &ScalePolicy,
5241        attempt: RunnerAttempt,
5242        waits: ReservationWaits,
5243    ) -> Result<RunnerAttempt, LifecycleError> {
5244        let id = attempt.id;
5245        let _bringing_up = BroughtUpOnDrop {
5246            attempts: &self.bringing_up,
5247            id,
5248        };
5249        let mut stopwatch = LaunchStopwatch::new(waits);
5250        let result = self.launch_steps(policy, attempt, &mut stopwatch).await;
5251        stopwatch.finish();
5252        self.ports.events.emit(AttemptEvent::Launched {
5253            attempt: id,
5254            started: result.is_ok(),
5255            timings: stopwatch.timings,
5256        });
5257        result
5258    }
5259
5260    /// Place and journal a new attempt: everything before its first external
5261    /// effect, and so before there is anything to time.
5262    fn allocate_attempt(&self, policy: &ScalePolicy) -> Result<RunnerAttempt, LifecycleError> {
5263        if !*self
5264            .recovery_complete
5265            .lock()
5266            .map_err(|_| LifecycleError::Journal)?
5267        {
5268            return Err(LifecycleError::RecoveryIncomplete);
5269        }
5270        let capability = self.ports.processes.probe(policy);
5271        if capability != ProviderCapability::Ready {
5272            return Err(LifecycleError::Failed(capability.refusal()));
5273        }
5274        let resolved = self.ports.processes.resolve(policy).map_err(|reason| {
5275            LifecycleError::Failed(if policy.execution_policy().is_native() {
5276                reason
5277            } else {
5278                safe_isolation_reason(reason, IsolationProviderFailure::RuntimeOperationFailed)
5279            })
5280        })?;
5281        if policy.execution_policy().is_native() != resolved.is_none() {
5282            return Err(LifecycleError::Failed(FailureReason::Other(
5283                "provider resolution mismatch".into(),
5284            )));
5285        }
5286        if let Some(resolution) = &resolved {
5287            if resolution.provider_kind == Backend::Auto {
5288                return Err(LifecycleError::Failed(FailureReason::Other(
5289                    "provider resolution mismatch".into(),
5290                )));
5291            }
5292            AttemptExecution::Isolated {
5293                provider_kind: resolution.provider_kind,
5294                environment_id: None,
5295                resolved_image: resolution.image.clone(),
5296                generation: "generation-check".into(),
5297            }
5298            .validate()
5299            .map_err(|_| {
5300                LifecycleError::Failed(FailureReason::Other(
5301                    "resolved image is incompatible".into(),
5302                ))
5303            })?;
5304        }
5305        policy
5306            .routing_labels()
5307            .ok_or(LifecycleError::Failed(FailureReason::JitRequestFailed))?;
5308        let id = AttemptId::new_random();
5309        let placement = self.allocate_workspace(policy, id)?;
5310        let mut attempt = RunnerAttempt::allocate_in(
5311            id,
5312            policy.id,
5313            placement.runtime,
5314            placement.workspace,
5315            self.ports.clock.now(),
5316        );
5317        if let Some(resolution) = &resolved {
5318            attempt
5319                .allocate_execution(AttemptExecution::Isolated {
5320                    provider_kind: resolution.provider_kind,
5321                    environment_id: None,
5322                    resolved_image: resolution.image.clone(),
5323                    generation: uuid::Uuid::new_v4().to_string(),
5324                })
5325                .map_err(|_| LifecycleError::Transition)?;
5326        }
5327        // This is deliberately the first effect after directory allocation, and
5328        // for a persistent attempt it is also what makes the slot lease durable
5329        // before any package or GitHub effect
5330        // (`02-target-architecture.md`, "Slot allocation", step 7).
5331        self.bringing_up
5332            .lock()
5333            .map_err(|_| LifecycleError::Journal)?
5334            .insert(id);
5335        if let Err(error) = self.record_allocation(&attempt) {
5336            self.bringing_up
5337                .lock()
5338                .map_err(|_| LifecycleError::Journal)?
5339                .remove(&id);
5340            return Err(error);
5341        }
5342        Ok(attempt)
5343    }
5344
5345    async fn launch_steps(
5346        &self,
5347        policy: &ScalePolicy,
5348        mut attempt: RunnerAttempt,
5349        stopwatch: &mut LaunchStopwatch,
5350    ) -> Result<RunnerAttempt, LifecycleError> {
5351        let id = attempt.id;
5352        let isolated = !attempt.execution().is_native();
5353        let Some(labels) = policy.routing_labels() else {
5354            return self.fail_launch(&mut attempt, FailureReason::JitRequestFailed);
5355        };
5356        stopwatch.begin(LaunchStep::Turn);
5357        // Never closed; held through the preparation below.
5358        let turn = self.turns.acquire().await.ok();
5359        stopwatch.begin(LaunchStep::Materialize);
5360        let package_timings = &mut stopwatch.timings.package;
5361        let version = match self
5362            .materialize_with_retry(policy, &attempt, package_timings)
5363            .await
5364        {
5365            Ok(version) => version,
5366            Err(reason) => return self.fail_launch(&mut attempt, reason),
5367        };
5368        self.versions
5369            .lock()
5370            .map_err(|_| LifecycleError::Journal)?
5371            .insert(id, version);
5372
5373        if isolated {
5374            attempt
5375                .begin_prepare(self.ports.clock.now())
5376                .map_err(|_| LifecycleError::Transition)?;
5377            self.record(&attempt)?;
5378        }
5379        stopwatch.begin(LaunchStep::Prepare);
5380        // Off the async thread: a provider may pull an image, and every target
5381        // of this host shares that thread.
5382        let preparing = {
5383            let (processes, attempt, policy) = (
5384                Arc::clone(&self.ports.processes),
5385                attempt.clone(),
5386                policy.clone(),
5387            );
5388            tokio::task::spawn_blocking(move || processes.prepare(&attempt, &policy)).await
5389        };
5390        drop(turn);
5391        let prepared = match preparing.unwrap_or(Err(FailureReason::ProcessStartFailed)) {
5392            Ok(prepared) => prepared,
5393            Err(reason) => {
5394                let safe_reason = if isolated {
5395                    safe_isolation_reason(reason, IsolationProviderFailure::RuntimeOperationFailed)
5396                } else {
5397                    reason
5398                };
5399                return self.fail_launch(&mut attempt, safe_reason);
5400            }
5401        };
5402        if isolated {
5403            let Some(identity) = prepared.identity() else {
5404                return Err(LifecycleError::Failed(FailureReason::IsolationProvider(
5405                    IsolationProviderFailure::OwnershipMismatch,
5406                )));
5407            };
5408            if !identity_matches_intent(self.host_id, &attempt, identity) {
5409                return Err(LifecycleError::Failed(FailureReason::IsolationProvider(
5410                    IsolationProviderFailure::OwnershipMismatch,
5411                )));
5412            }
5413            let EnvironmentIdentity::Isolated { environment_id, .. } = identity else {
5414                unreachable!()
5415            };
5416            attempt
5417                .prepared_environment(environment_id.clone())
5418                .map_err(|_| LifecycleError::Transition)?;
5419            attempt
5420                .mark_prepared(self.ports.clock.now())
5421                .map_err(|_| LifecycleError::Transition)?;
5422            self.record(&attempt)?;
5423        }
5424
5425        stopwatch.begin(LaunchStep::Register);
5426        let jit_request =
5427            JitRunnerRequest::for_policy(runner_name(id), self.runner_group_id, labels);
5428        let registration = match self.register_with_retry(policy, id, &jit_request).await {
5429            Ok(registration) => registration,
5430            Err(error) => return self.fail_launch(&mut attempt, error.reason()),
5431        };
5432        stopwatch.begin(LaunchStep::Spawn);
5433        let runner_id = registration.runner().id;
5434        write_runner_id(attempt.runtime_path(), runner_id)?;
5435        attempt
5436            .jit_received(self.ports.clock.now())
5437            .map_err(|_| LifecycleError::Transition)?;
5438        self.record(&attempt)?;
5439        let config = registration.into_config();
5440        if isolated {
5441            let started = self
5442                .ports
5443                .processes
5444                .start(prepared, &attempt, OneTimeJitHandoff::new(&config))
5445                .map_err(|failure| {
5446                    LifecycleError::Failed(safe_isolation_reason(
5447                        failure.reason,
5448                        IsolationProviderFailure::RuntimeOperationFailed,
5449                    ))
5450                })?;
5451            if !identity_matches_intent(self.host_id, &attempt, &started) {
5452                return Err(LifecycleError::Failed(FailureReason::IsolationProvider(
5453                    IsolationProviderFailure::OwnershipMismatch,
5454                )));
5455            }
5456            attempt
5457                .started_isolated(self.ports.clock.now())
5458                .map_err(|_| LifecycleError::Transition)?;
5459            self.record(&attempt)?;
5460            return Ok(attempt);
5461        }
5462        let mut issued = 0_u32;
5463        let pid = loop {
5464            issued = issued.saturating_add(1);
5465            match self.ports.processes.start(
5466                prepared.clone(),
5467                &attempt,
5468                OneTimeJitHandoff::new(&config),
5469            ) {
5470                Ok(EnvironmentIdentity::NativeProcess(pid)) => break pid,
5471                Ok(EnvironmentIdentity::Isolated { .. }) => {
5472                    // No isolated launch sequence is wired in this task. Keep
5473                    // the nonterminal JIT journal as a capacity fence rather
5474                    // than treating an unexpected resource as a native PID.
5475                    return Err(LifecycleError::Failed(FailureReason::Other(
5476                        "unexpected isolated environment identity".into(),
5477                    )));
5478                }
5479                Err(error) => {
5480                    if let Some(pid) = error.live_pid {
5481                        attempt
5482                            .started(pid, self.ports.clock.now())
5483                            .map_err(|_| LifecycleError::Transition)?;
5484                        self.record(&attempt)?;
5485                        return Err(LifecycleError::Failed(error.reason));
5486                    }
5487                    if !error.retryable
5488                        || issued >= self.retry.max_attempts.max(1)
5489                        || !self.ports.demand.persists(policy.id).await
5490                    {
5491                        return self.fail_launch(&mut attempt, error.reason);
5492                    }
5493                    let delay = self.retry.delay(issued);
5494                    self.ports.events.emit(AttemptEvent::Retry {
5495                        attempt: attempt.id,
5496                        operation: "process_start",
5497                        delay,
5498                    });
5499                    self.ports.delay.wait(delay).await;
5500                    if !self.ports.demand.persists(policy.id).await {
5501                        return self.fail_launch(&mut attempt, error.reason);
5502                    }
5503                }
5504            }
5505        };
5506        attempt
5507            .started(pid, self.ports.clock.now())
5508            .map_err(|_| LifecycleError::Transition)?;
5509        self.record(&attempt)?;
5510        Ok(attempt)
5511    }
5512
5513    fn fail_launch<T>(
5514        &self,
5515        attempt: &mut RunnerAttempt,
5516        reason: FailureReason,
5517    ) -> Result<T, LifecycleError> {
5518        self.conclude(attempt, AttemptOutcome::failed(reason.clone()))?;
5519        Err(LifecycleError::Failed(reason))
5520    }
5521
5522    /// e2's prune guard is invoked only with e1's allocation guard borrowed.
5523    /// The otherwise-unused argument is a compile-time witness of the ordering.
5524    fn prune_under_allocation_lock(&self, guard: &AllocationGuard) -> Result<(), LifecycleError> {
5525        let attempts = self
5526            .ports
5527            .store
5528            .attempts()
5529            .map_err(|_| LifecycleError::Journal)?;
5530        self.ports
5531            .packages
5532            .prune_obsolete_guarded(PruneAuthority::from_launch_request(guard), &attempts)
5533            .map_err(LifecycleError::Failed)?;
5534        #[cfg(target_os = "macos")]
5535        self.prune_abandoned_seeds(&attempts);
5536        Ok(())
5537    }
5538
5539    /// Remove the package seed from every runner root this host once placed a
5540    /// runtime in and no longer does: a changed host root override, a removed
5541    /// persistent policy, or one that moved its workspace.
5542    ///
5543    /// The roots ever used are the parents of the journal's runtime paths; the
5544    /// roots in use are the effective host root and every persistent policy's
5545    /// root. When the effective host root cannot be resolved nothing is
5546    /// removed, because a seed still in use cannot then be told apart. No
5547    /// launch clones from a seed under a root no policy uses, and it is
5548    /// best-effort: a seed left behind costs disk, never a launch.
5549    #[cfg(target_os = "macos")]
5550    fn prune_abandoned_seeds(&self, attempts: &[RunnerAttempt]) {
5551        let Ok(policies) = self.ports.store.policies() else {
5552            return;
5553        };
5554        let Ok(Some(host_root)) = self.resolvable_host_root() else {
5555            return;
5556        };
5557        let in_use: BTreeSet<PathBuf> = policies
5558            .iter()
5559            .filter_map(|policy| match policy.workspace_policy() {
5560                WorkspacePolicy::Persistent { root } => Some(canonical_or_raw(root.as_path())),
5561                WorkspacePolicy::Ephemeral => None,
5562            })
5563            .chain(std::iter::once(canonical_or_raw(host_root.as_path())))
5564            .collect();
5565        // Distinct parents first: the journal keeps every attempt ever made.
5566        let used: BTreeSet<&Path> = attempts
5567            .iter()
5568            .filter_map(|attempt| attempt.runtime_path().parent())
5569            .collect();
5570        for root in used.into_iter().map(canonical_or_raw) {
5571            if !in_use.contains(&root) {
5572                let _ = remove_runtime_tree(&root.join(PACKAGE_SEED_DIR));
5573            }
5574        }
5575    }
5576}
5577
5578#[cfg(target_os = "macos")]
5579fn canonical_or_raw(path: &Path) -> PathBuf {
5580    fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
5581}
5582
5583#[async_trait]
5584impl RunnerLauncher for LifecycleLauncher {
5585    async fn host_capacity(
5586        &self,
5587        host: &runner_manager_domain::model::Host,
5588    ) -> Result<std::num::NonZeroU16, crate::reconcile::HostCapacityError> {
5589        self.ports
5590            .store
5591            .host(host.id)
5592            .map_err(|_| crate::reconcile::HostCapacityError::JournalUnreadable)?
5593            .map(|current| current.host_capacity)
5594            .ok_or(crate::reconcile::HostCapacityError::MissingHost)
5595    }
5596
5597    async fn supervise(
5598        &self,
5599        policy: &ScalePolicy,
5600    ) -> Result<Vec<ReplacementIntent>, LaunchFailure> {
5601        LifecycleLauncher::supervise(self, policy)
5602            .await
5603            .map_err(|error| LaunchFailure::new(error.reason()))
5604    }
5605
5606    async fn attempts(&self) -> Result<Vec<RunnerAttempt>, LaunchFailure> {
5607        self.ports.store.attempts().map_err(|_| {
5608            LaunchFailure::new(FailureReason::Other(
5609                "attempt journal could not be read".into(),
5610            ))
5611        })
5612    }
5613
5614    async fn launch(&self, request: LaunchRequest<'_>) -> Result<RunnerAttempt, LaunchFailure> {
5615        self.reserve_attempt(request.policy, request.allocation_guard)
5616            .map_err(|error| LaunchFailure::new(error.reason()))
5617    }
5618
5619    async fn finish_launch(
5620        &self,
5621        policy: &ScalePolicy,
5622        attempt: RunnerAttempt,
5623        waits: ReservationWaits,
5624    ) -> Result<RunnerAttempt, LaunchFailure> {
5625        self.finish_attempt(policy, attempt, waits)
5626            .await
5627            .map_err(|error| LaunchFailure::new(error.reason()))
5628    }
5629
5630    async fn clean(&self, id: AttemptId) -> Result<(), LaunchFailure> {
5631        let mut attempt = self
5632            .ports
5633            .store
5634            .attempt(id)
5635            .map_err(|_| {
5636                LaunchFailure::new(FailureReason::Other(
5637                    "attempt journal could not be read".into(),
5638                ))
5639            })?
5640            .ok_or_else(|| {
5641                LaunchFailure::new(FailureReason::Other(
5642                    "attempt disappeared from the journal".into(),
5643                ))
5644            })?;
5645        self.clean_attempt(&mut attempt)
5646            .map_err(|error| LaunchFailure::new(error.reason()))?;
5647        self.cleanup_retries().succeeded(id);
5648        Ok(())
5649    }
5650}
5651
5652/// The name an attempt's runner registers with GitHub, which a
5653/// `workflow_job` webhook's `runner_name` repeats.
5654#[must_use]
5655pub fn runner_name(attempt: AttemptId) -> String {
5656    format!("runner-manager-{attempt}")
5657}
5658
5659/// How long reading a runner's diagnostics may take.
5660const EVIDENCE_DEADLINE: Duration = Duration::from_secs(2);
5661
5662/// What a native runner's own diagnostics say about whether it ran a job.
5663///
5664/// `Runner.Listener` writes `_diag/Runner_*.log` when it starts, and starts
5665/// `Runner.Worker` -- which writes `_diag/Worker_*.log` -- only for a job it has
5666/// acquired. So a worker log is proof a job ran, and a listener log without one
5667/// is proof none did. With neither, the directory is not where the listener
5668/// wrote, and nothing is concluded from it. `_diag` is scrubbed with the rest of
5669/// the runtime after every attempt, persistent slots included, so a log found
5670/// here belongs to this attempt.
5671#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5672enum JobEvidence {
5673    RanAJob,
5674    NoJob,
5675    Unknown,
5676}
5677
5678impl JobEvidence {
5679    fn of(runtime: &Path) -> Self {
5680        let diag = runtime.join("_diag");
5681        // Bounded: the daemon's loop runs on one thread, and a runtime on a
5682        // volume that does not answer (a hung disk, a pending macOS question)
5683        // must not stall it for longer than this.
5684        if runner_manager_platform::host_fitness::directory_responds(&diag, EVIDENCE_DEADLINE)
5685            != runner_manager_platform::host_fitness::Responsiveness::Responds
5686        {
5687            return Self::Unknown;
5688        }
5689        let Ok(entries) = fs::read_dir(diag) else {
5690            return Self::Unknown;
5691        };
5692        let mut listener = false;
5693        for entry in entries.flatten() {
5694            let name = entry.file_name();
5695            let name = name.to_string_lossy();
5696            if !name.ends_with(".log") {
5697                continue;
5698            }
5699            if name.starts_with("Worker_") {
5700                return Self::RanAJob;
5701            }
5702            listener |= name.starts_with("Runner_");
5703        }
5704        if listener { Self::NoJob } else { Self::Unknown }
5705    }
5706}
5707
5708/// The attempt's GitHub runner id: journalled, observed, or from the sidecar
5709/// written at registration.
5710fn known_runner_id(attempt: &RunnerAttempt, observed: Option<u64>) -> Option<u64> {
5711    attempt
5712        .github_runner_id()
5713        .or(observed)
5714        .or_else(|| read_runner_id(attempt.runtime_path()))
5715}
5716
5717fn read_runner_id(runtime: &Path) -> Option<u64> {
5718    fs::read_to_string(runtime.join(RUNNER_ID_FILE))
5719        .ok()?
5720        .trim()
5721        .parse()
5722        .ok()
5723}
5724
5725fn write_runner_id(runtime: &Path, runner_id: u64) -> Result<(), LifecycleError> {
5726    let target = runtime.join(RUNNER_ID_FILE);
5727    if let Some(existing) = read_runner_id(runtime) {
5728        return (existing == runner_id)
5729            .then_some(())
5730            .ok_or(LifecycleError::Journal);
5731    }
5732    let temporary = runtime.join(format!("{RUNNER_ID_FILE}.{}.tmp", uuid::Uuid::new_v4()));
5733    write_durable_file(&temporary, runner_id.to_string().as_bytes())
5734        .map_err(|_| LifecycleError::Journal)?;
5735    match fs::rename(&temporary, &target) {
5736        Ok(()) => sync_directory(runtime).map_err(|_| LifecycleError::Journal),
5737        Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
5738            let _ = fs::remove_file(&temporary);
5739            (read_runner_id(runtime) == Some(runner_id))
5740                .then_some(())
5741                .ok_or(LifecycleError::Journal)
5742        }
5743        Err(_) => {
5744            let _ = fs::remove_file(&temporary);
5745            Err(LifecycleError::Journal)
5746        }
5747    }
5748}
5749
5750fn write_durable_file(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
5751    let mut options = fs::OpenOptions::new();
5752    options.create(true).truncate(true).write(true);
5753    // Never through a link: the attempt directory belongs to the runner's
5754    // account when that is not this process's, and a link a job left at this
5755    // name would otherwise turn the write onto any file this process can reach.
5756    #[cfg(unix)]
5757    std::os::unix::fs::OpenOptionsExt::custom_flags(&mut options, libc::O_NOFOLLOW);
5758    let mut file = options.open(path)?;
5759    file.write_all(bytes)?;
5760    file.sync_all()?;
5761    let parent = path.parent().ok_or_else(|| {
5762        std::io::Error::new(
5763            std::io::ErrorKind::InvalidInput,
5764            "file has no parent directory",
5765        )
5766    })?;
5767    sync_directory(parent)
5768}
5769
5770#[cfg(unix)]
5771fn sync_directory(path: &Path) -> std::io::Result<()> {
5772    fs::File::open(path)?.sync_all()
5773}
5774
5775#[cfg(windows)]
5776fn sync_directory(path: &Path) -> std::io::Result<()> {
5777    use std::os::windows::fs::OpenOptionsExt;
5778
5779    const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
5780    const FILE_SHARE_ALL: u32 = 0x0000_0007;
5781    const GENERIC_WRITE: u32 = 0x4000_0000;
5782    fs::OpenOptions::new()
5783        .access_mode(GENERIC_WRITE)
5784        .share_mode(FILE_SHARE_ALL)
5785        .custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
5786        .open(path)?
5787        .sync_all()
5788}
5789
5790#[cfg(test)]
5791mod tests {
5792    use super::*;
5793
5794    /// A native attempt whose directory, `name` under `root`, already exists.
5795    #[cfg(unix)]
5796    fn attempt_owned_in(root: &Path, name: &str) -> RunnerAttempt {
5797        let runtime = root.join(name);
5798        fs::create_dir_all(&runtime).unwrap();
5799        RunnerAttempt::allocate(
5800            AttemptId::new_random(),
5801            PolicyId::from_u128(0xfeed),
5802            &runtime,
5803            FakeClock::default().now(),
5804        )
5805    }
5806
5807    #[cfg(unix)]
5808    #[test]
5809    fn no_two_live_attempts_share_a_runner_account() {
5810        use std::os::unix::fs::MetadataExt as _;
5811
5812        // This test cannot hand anything over, so every directory it makes is
5813        // its own: its uid plays the account that owns them all.
5814        let root = tempfile::tempdir().unwrap();
5815        let me = fs::metadata(root.path()).unwrap();
5816        let mine = RunnerAccount {
5817            uid: me.uid(),
5818            gid: me.gid(),
5819        };
5820        let other = RunnerAccount {
5821            uid: me.uid().wrapping_add(1),
5822            gid: me.gid(),
5823        };
5824        let processes = NativeProcesses {
5825            runner: RunnerIdentity::Pool(RunnerPool {
5826                gid: me.gid(),
5827                accounts: vec![other, mine],
5828                legacy: None,
5829            }),
5830            ..NativeProcesses::default()
5831        };
5832
5833        let first = attempt_owned_in(root.path(), "a");
5834        assert_eq!(
5835            processes.account_for(&first).unwrap(),
5836            Some(mine),
5837            "an attempt an account already owns keeps it"
5838        );
5839        assert_eq!(processes.account_for(&first).unwrap(), Some(mine));
5840        let second = attempt_owned_in(root.path(), "b");
5841        assert_eq!(
5842            processes.account_for(&second).unwrap(),
5843            Some(other),
5844            "the account a live attempt holds is not given to another"
5845        );
5846        let third = attempt_owned_in(root.path(), "c");
5847        assert!(
5848            processes.account_for(&third).is_err(),
5849            "with every account busy no runner starts on a shared one"
5850        );
5851
5852        processes.destroy(&first).unwrap();
5853        fs::remove_dir(first.runtime_path()).unwrap();
5854        assert_eq!(processes.account_for(&third).unwrap(), Some(mine));
5855    }
5856
5857    #[cfg(unix)]
5858    #[test]
5859    fn a_pool_with_only_the_earlier_single_account_shares_it_as_before() {
5860        use std::os::unix::fs::MetadataExt as _;
5861
5862        let root = tempfile::tempdir().unwrap();
5863        let me = fs::metadata(root.path()).unwrap();
5864        let legacy = RunnerAccount {
5865            uid: me.uid(),
5866            gid: me.gid(),
5867        };
5868        let processes = NativeProcesses {
5869            runner: RunnerIdentity::Pool(RunnerPool {
5870                gid: me.gid(),
5871                accounts: Vec::new(),
5872                legacy: Some(legacy),
5873            }),
5874            ..NativeProcesses::default()
5875        };
5876        for name in ["a", "b"] {
5877            assert_eq!(
5878                processes
5879                    .account_for(&attempt_owned_in(root.path(), name))
5880                    .unwrap(),
5881                Some(legacy)
5882            );
5883        }
5884    }
5885
5886    #[test]
5887    fn a_confined_root_without_the_pool_starts_no_runner() {
5888        let processes = NativeProcesses {
5889            runner: RunnerIdentity::Missing("no pool".into()),
5890            ..NativeProcesses::default()
5891        };
5892        let attempt = RunnerAttempt::allocate(
5893            AttemptId::new_random(),
5894            PolicyId::from_u128(0xfeed),
5895            Path::new("/nonexistent/attempt"),
5896            FakeClock::default().now(),
5897        );
5898        assert_eq!(
5899            processes.account_for(&attempt),
5900            Err(FailureReason::Other("no pool".into()))
5901        );
5902    }
5903    use std::collections::BTreeSet;
5904    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
5905
5906    use crate::reconcile::{AllocationLock, InProcessAllocationLock};
5907    use runner_manager_domain::model::{Elapsed, TargetScope};
5908    use runner_manager_domain::store::SqliteStore;
5909    use runner_manager_github::jit::JitRunner;
5910    use runner_manager_testkit::clock::FakeClock;
5911    use runner_manager_testkit::fixtures;
5912
5913    #[test]
5914    fn production_provider_set_routes_every_implemented_isolated_backend() {
5915        let providers = PlatformExecutionProvider::new(HostId::from_u128(1));
5916        assert!(providers.isolated_provider(Backend::Auto).is_some());
5917        assert!(providers.isolated_provider(Backend::Oci).is_some());
5918        assert!(
5919            providers
5920                .isolated_provider(Backend::WindowsHyperVContainer)
5921                .is_some()
5922        );
5923        assert!(
5924            providers
5925                .isolated_provider(Backend::VirtualMachine)
5926                .is_none()
5927        );
5928    }
5929
5930    /// One event's fields, in the order they were recorded.
5931    type CapturedFields = Vec<(String, String)>;
5932
5933    /// Keeps the `(name, value)` fields of every `tracing` event emitted while
5934    /// it is installed.
5935    ///
5936    /// The names are kept and not just the rendered line, so a test can hold the
5937    /// event to `crate::logging`'s two rules -- the field allow-list and the
5938    /// value scrub -- instead of asserting that some string was passed to a
5939    /// macro.
5940    #[derive(Clone, Default)]
5941    struct CapturedEvents(std::sync::Arc<std::sync::Mutex<Vec<CapturedFields>>>);
5942
5943    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for CapturedEvents {
5944        fn on_event(
5945            &self,
5946            event: &tracing::Event<'_>,
5947            _context: tracing_subscriber::layer::Context<'_, S>,
5948        ) {
5949            struct Collect(Vec<(String, String)>);
5950            impl tracing::field::Visit for Collect {
5951                fn record_debug(
5952                    &mut self,
5953                    field: &tracing::field::Field,
5954                    value: &dyn std::fmt::Debug,
5955                ) {
5956                    // `{value:?}` on a `&str` field would keep the quotes, and
5957                    // the redaction rules are about the value, not its literal.
5958                    self.0.push((
5959                        field.name().to_owned(),
5960                        format!("{value:?}").trim_matches('"').to_owned(),
5961                    ));
5962                }
5963            }
5964            let mut collected = Collect(Vec::new());
5965            event.record(&mut collected);
5966            self.0
5967                .lock()
5968                .expect("the capture mutex is not poisoned")
5969                .push(collected.0);
5970        }
5971    }
5972
5973    /// The regression behind a three-hour outage that showed nothing an operator
5974    /// could act on.
5975    ///
5976    /// A root the daemon cannot use refuses the launch *before*
5977    /// `record_allocation`, so no attempt row is ever written: `b2` has nothing
5978    /// to carry the failure on and `g2` has nothing to show it from. The
5979    /// lifecycle event keeps only the variant -- `failure_reason_kind` allows no
5980    /// free text on an event -- so the whole failure read
5981    /// `runner_start_failed reason=other`, once per poll, for hours.
5982    ///
5983    /// The assertion is deliberately made against the **production redaction
5984    /// rules** and not merely against what was emitted. An earlier attempt at
5985    /// this fix logged the rendered error on a `detail` field and passed a test
5986    /// exactly like this one, while shipping `detail="[redacted]"`: the field
5987    /// was not allow-listed, and had it been, the value is mostly paths and
5988    /// would have been scrubbed to `[path]`. A test that does not ask
5989    /// `crate::logging` what survives is a test that proves nothing.
5990    #[test]
5991    fn a_launch_the_runner_root_refused_names_the_cause_in_the_log_that_ships() {
5992        use runner_manager_platform::logging;
5993        use tracing_subscriber::layer::SubscriberExt as _;
5994
5995        let captured = CapturedEvents::default();
5996        let error = RunnerRootError::DeniedByPrivacyPolicy {
5997            requested: PathBuf::from("/Volumes/NVME/runners"),
5998            refused: PathBuf::from("/Volumes/NVME"),
5999            remediation: RootOwner::Host.remediation(),
6000        };
6001        let kind = error.kind();
6002
6003        let failure = tracing::subscriber::with_default(
6004            tracing_subscriber::registry().with(captured.clone()),
6005            || root_failure(error),
6006        );
6007
6008        // The reason still carries the whole sentence. Nothing on *this* path
6009        // renders it -- there is no attempt row, and the event carries only the
6010        // variant -- so this is a guard against a refactor that drops the detail
6011        // before some future surface can show it, and not a claim that one does.
6012        assert!(
6013            matches!(
6014                &failure,
6015                LifecycleError::Failed(FailureReason::Other(detail))
6016                    if detail.contains("/Volumes/NVME/runners")
6017                        && detail.contains("Full Disk Access")
6018            ),
6019            "the reason must still carry the detail: {failure:?}"
6020        );
6021
6022        let events = captured
6023            .0
6024            .lock()
6025            .expect("the capture mutex is not poisoned")
6026            .clone();
6027        let event = events
6028            .iter()
6029            .find(|fields| fields.iter().any(|(_, value)| value.contains(kind)))
6030            .unwrap_or_else(|| panic!("the refusal did not name its cause: {events:?}"));
6031
6032        // Held to the rules the real sink applies, by name and by value. An
6033        // earlier fix put the rendered error on an unlisted `detail` field and
6034        // shipped `[redacted]`; asserting only that something was emitted would
6035        // have passed then too.
6036        for (name, value) in event {
6037            assert!(
6038                logging::is_field_allowed(name),
6039                "`{name}` is not allow-listed, so it ships as `{}`: {event:?}",
6040                logging::REDACTION
6041            );
6042            assert_eq!(
6043                &logging::redact(value),
6044                value,
6045                "`{name}` does not survive value-shape scrubbing: {event:?}"
6046            );
6047        }
6048    }
6049
6050    /// A slot number, for the tests that name one.
6051    fn nz(slot: u16) -> NonZeroU16 {
6052        NonZeroU16::new(slot).expect("a positive slot")
6053    }
6054
6055    const JIT: &str = "eyJzZWNyZXQiOiJnaHBfRE9fTk9UX0xFQUsifQ==";
6056
6057    #[derive(Debug, Default)]
6058    struct FakeGithubLifecycle {
6059        registration_failures: Mutex<VecDeque<bool>>,
6060        observations: Mutex<VecDeque<LifecycleGithubObservation>>,
6061        registrations: AtomicUsize,
6062        remaining_runners: AtomicUsize,
6063        /// Every runner id `deregister` was asked to remove, in order. A count
6064        /// would not do: the assertions worth making are that the *right*
6065        /// registration was deleted and that it was deleted once.
6066        deregistrations: Mutex<Vec<u64>>,
6067        /// Set to make `deregister` answer `false`, standing for a GitHub that
6068        /// could not be reached at the moment the attempt concluded.
6069        deregistration_fails: AtomicBool,
6070        /// The journal to read *during* a registration, for the ordering
6071        /// assertion `02-target-architecture.md` makes: the slot lease is
6072        /// written "before package or GitHub effects". Reading it afterwards
6073        /// would pass even if the write happened second.
6074        journal: Mutex<Option<Arc<SqliteStore>>>,
6075        /// One entry per registration, in order.
6076        registration_facts: Mutex<Vec<RegistrationFact>>,
6077    }
6078
6079    /// What one JIT registration saw of the world at the moment it was issued.
6080    ///
6081    /// `runner_name` is what ties the other two fields to *one* attempt: with
6082    /// two allocators racing, "some slot was journalled" is a much weaker claim
6083    /// than "the slot this very request belongs to was journalled", and only the
6084    /// name distinguishes them.
6085    #[derive(Debug, Clone)]
6086    struct RegistrationFact {
6087        /// The persistent slots the journal already held.
6088        leased_slots: Vec<u16>,
6089        /// The `work_folder` the request carried.
6090        work_folder: String,
6091        /// The runner name the request carried, i.e. [`runner_name`] of the
6092        /// registering attempt.
6093        runner_name: String,
6094        journal_states: Vec<AttemptState>,
6095    }
6096
6097    impl FakeGithubLifecycle {
6098        fn fail(mut self, terminal: bool) -> Self {
6099            self.registration_failures
6100                .get_mut()
6101                .expect("unpoisoned")
6102                .push_back(terminal);
6103            self
6104        }
6105
6106        fn watch_journal(&self, store: Arc<SqliteStore>) {
6107            *self.journal.lock().unwrap() = Some(store);
6108        }
6109
6110        fn registration_facts(&self) -> Vec<RegistrationFact> {
6111            self.registration_facts.lock().unwrap().clone()
6112        }
6113
6114        fn observe(&self, observation: GithubRunnerObservation) {
6115            let observation = match observation {
6116                GithubRunnerObservation::Unreachable => LifecycleGithubObservation::unreachable(),
6117                GithubRunnerObservation::NotRegistered => {
6118                    LifecycleGithubObservation::not_registered()
6119                }
6120                GithubRunnerObservation::Registered { busy } => {
6121                    LifecycleGithubObservation::registered(73, busy)
6122                }
6123            };
6124            self.observations.lock().unwrap().push_back(observation);
6125        }
6126    }
6127
6128    #[async_trait]
6129    impl LifecycleGithub for FakeGithubLifecycle {
6130        async fn register(
6131            &self,
6132            _target: &ScaleTarget,
6133            request: &JitRunnerRequest,
6134            _cancel: &CancelToken,
6135        ) -> Result<JitRegistration, JitRequestFailure> {
6136            self.registrations.fetch_add(1, Ordering::SeqCst);
6137            if let Some(store) = self.journal.lock().unwrap().as_ref() {
6138                let attempts = store.attempts().expect("the journal is readable");
6139                let slots = attempts
6140                    .iter()
6141                    .filter_map(|attempt| attempt.workspace().slot_number())
6142                    .collect();
6143                self.registration_facts
6144                    .lock()
6145                    .unwrap()
6146                    .push(RegistrationFact {
6147                        leased_slots: slots,
6148                        work_folder: request.work_folder().to_string(),
6149                        runner_name: request.name().to_string(),
6150                        journal_states: attempts.iter().map(RunnerAttempt::state).collect(),
6151                    });
6152            }
6153            if let Some(terminal) = self.registration_failures.lock().unwrap().pop_front() {
6154                return Err(JitRequestFailure {
6155                    terminal,
6156                    reason: if terminal {
6157                        FailureReason::Other("GitHub refused JIT registration with 403".into())
6158                    } else {
6159                        FailureReason::JitRequestFailed
6160                    },
6161                    retry_after: None,
6162                });
6163            }
6164            self.remaining_runners.store(1, Ordering::SeqCst);
6165            Ok(JitRegistration::new(
6166                EncodedJitConfig::new(JIT),
6167                JitRunner {
6168                    id: 73,
6169                    name: request.name().to_string(),
6170                    os: "windows".into(),
6171                    status: "offline".into(),
6172                    busy: false,
6173                    runner_group_id: Some(1),
6174                    labels: request.labels().to_vec(),
6175                },
6176            ))
6177        }
6178
6179        async fn observe(
6180            &self,
6181            _target: &ScaleTarget,
6182            _attempt: AttemptId,
6183            _cancel: &CancelToken,
6184        ) -> LifecycleGithubObservation {
6185            let observation = self
6186                .observations
6187                .lock()
6188                .unwrap()
6189                .pop_front()
6190                .unwrap_or(LifecycleGithubObservation::not_registered());
6191            if observation.status == GithubRunnerObservation::NotRegistered {
6192                self.remaining_runners.store(0, Ordering::SeqCst);
6193            }
6194            observation
6195        }
6196
6197        async fn deregister(
6198            &self,
6199            _target: &ScaleTarget,
6200            runner_id: u64,
6201            _cancel: &CancelToken,
6202        ) -> bool {
6203            self.deregistrations.lock().unwrap().push(runner_id);
6204            if self.deregistration_fails.load(Ordering::SeqCst) {
6205                return false;
6206            }
6207            self.remaining_runners.store(0, Ordering::SeqCst);
6208            true
6209        }
6210    }
6211
6212    #[derive(Debug)]
6213    struct FakePackages {
6214        version: RunnerVersion,
6215        leases: Mutex<BTreeSet<AttemptId>>,
6216        materializations: AtomicUsize,
6217        materializing: AtomicUsize,
6218        peak_materializing: AtomicUsize,
6219        materialization_failures: AtomicUsize,
6220        release_failures: AtomicUsize,
6221        releases: AtomicUsize,
6222        prunes: AtomicUsize,
6223    }
6224
6225    impl Default for FakePackages {
6226        fn default() -> Self {
6227            Self {
6228                version: RunnerVersion::parse("2.330.0").unwrap(),
6229                leases: Mutex::new(BTreeSet::new()),
6230                materializations: AtomicUsize::new(0),
6231                materializing: AtomicUsize::new(0),
6232                peak_materializing: AtomicUsize::new(0),
6233                materialization_failures: AtomicUsize::new(0),
6234                release_failures: AtomicUsize::new(0),
6235                releases: AtomicUsize::new(0),
6236                prunes: AtomicUsize::new(0),
6237            }
6238        }
6239    }
6240
6241    impl FakePackages {
6242        fn fail_materializations(&self, count: usize) {
6243            self.materialization_failures.store(count, Ordering::SeqCst);
6244        }
6245
6246        fn fail_releases(&self, count: usize) {
6247            self.release_failures.store(count, Ordering::SeqCst);
6248        }
6249    }
6250
6251    #[async_trait]
6252    impl RuntimePackages for FakePackages {
6253        async fn materialize(
6254            &self,
6255            attempt: &RunnerAttempt,
6256            _timings: &mut PackageTimings,
6257        ) -> Result<RunnerVersion, FailureReason> {
6258            self.materializations.fetch_add(1, Ordering::SeqCst);
6259            let now = self.materializing.fetch_add(1, Ordering::SeqCst) + 1;
6260            self.peak_materializing.fetch_max(now, Ordering::SeqCst);
6261            for _ in 0..5 {
6262                tokio::task::yield_now().await;
6263            }
6264            self.materializing.fetch_sub(1, Ordering::SeqCst);
6265            if self
6266                .materialization_failures
6267                .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
6268                    if left > 0 { Some(left - 1) } else { None }
6269                })
6270                .is_ok()
6271            {
6272                return Err(FailureReason::Other(
6273                    "runner package materialization failed transiently".into(),
6274                ));
6275            }
6276            fs::create_dir_all(attempt.runtime_path()).unwrap();
6277            fs::write(attempt.runtime_path().join("runner-package"), b"verified").unwrap();
6278            self.leases.lock().unwrap().insert(attempt.id);
6279            Ok(self.version.clone())
6280        }
6281
6282        fn release(&self, attempt: AttemptId) -> Result<(), FailureReason> {
6283            if self
6284                .release_failures
6285                .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
6286                    if left > 0 { Some(left - 1) } else { None }
6287                })
6288                .is_ok()
6289            {
6290                return Err(FailureReason::Other(
6291                    "runner package lease could not be released".into(),
6292                ));
6293            }
6294            self.leases.lock().unwrap().remove(&attempt);
6295            self.releases.fetch_add(1, Ordering::SeqCst);
6296            Ok(())
6297        }
6298
6299        fn prune_obsolete_guarded(
6300            &self,
6301            _authority: PruneAuthority<'_>,
6302            _attempts: &[RunnerAttempt],
6303        ) -> Result<(), FailureReason> {
6304            self.prunes.fetch_add(1, Ordering::SeqCst);
6305            Ok(())
6306        }
6307    }
6308
6309    #[derive(Debug, Default)]
6310    struct FakeProcesses {
6311        alive: AtomicBool,
6312        completed_successfully: AtomicBool,
6313        spawns: AtomicUsize,
6314        spawn_failures: AtomicUsize,
6315        live_spawn_failure: AtomicBool,
6316        terminations: AtomicUsize,
6317        intent: AtomicBool,
6318        intent_failure: AtomicBool,
6319        actions: Mutex<Vec<&'static str>>,
6320        saw_secret: AtomicBool,
6321    }
6322
6323    impl FakeProcesses {
6324        fn fail_spawns(&self, count: usize) {
6325            self.spawn_failures.store(count, Ordering::SeqCst);
6326        }
6327
6328        fn fail_spawn_with_live_child(&self) {
6329            self.live_spawn_failure.store(true, Ordering::SeqCst);
6330        }
6331
6332        fn set_alive(&self, alive: bool) {
6333            self.alive.store(alive, Ordering::SeqCst);
6334        }
6335
6336        fn finish_successfully(&self) {
6337            self.completed_successfully.store(true, Ordering::SeqCst);
6338            self.alive.store(false, Ordering::SeqCst);
6339        }
6340
6341        fn fail_intent(&self) {
6342            self.intent_failure.store(true, Ordering::SeqCst);
6343        }
6344    }
6345
6346    impl ProcessSupervisor for FakeProcesses {
6347        fn spawn(
6348            &self,
6349            attempt: &RunnerAttempt,
6350            config: &EncodedJitConfig,
6351        ) -> Result<u32, ProcessStartFailure> {
6352            self.spawns.fetch_add(1, Ordering::SeqCst);
6353            // Model the production handoff on both paths: the sensitive file is
6354            // scoped to this call and absent when it returns.
6355            let handoff = RestrictiveHandoff::create(
6356                attempt.runtime_path(),
6357                SecretString::from(config.expose().to_owned()),
6358            )
6359            .unwrap();
6360            self.saw_secret
6361                .store(config.expose() == JIT, Ordering::SeqCst);
6362            let handoff_path = handoff.path().to_path_buf();
6363            let failing = self
6364                .spawn_failures
6365                .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
6366                    if left > 0 { Some(left - 1) } else { None }
6367                })
6368                .is_ok();
6369            drop(handoff);
6370            assert!(!handoff_path.exists(), "handoff must be absent on return");
6371            if self.live_spawn_failure.swap(false, Ordering::SeqCst) {
6372                self.alive.store(true, Ordering::SeqCst);
6373                return Err(ProcessStartFailure::after_spawn_live(4242));
6374            }
6375            if failing {
6376                return Err(ProcessStartFailure::before_spawn(
6377                    FailureReason::ProcessStartFailed,
6378                ));
6379            }
6380            self.alive.store(true, Ordering::SeqCst);
6381            Ok(4242)
6382        }
6383
6384        fn is_alive(&self, _attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
6385            self.actions.lock().unwrap().push("observe_process");
6386            Ok(self.alive.load(Ordering::SeqCst))
6387        }
6388
6389        fn recovered_pid(&self, _attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
6390            Ok(self.alive.load(Ordering::SeqCst).then_some(4242))
6391        }
6392
6393        fn completed_successfully(&self, _attempt: &RunnerAttempt) -> bool {
6394            self.completed_successfully.load(Ordering::SeqCst)
6395        }
6396
6397        fn record_terminate_intent(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6398            self.actions.lock().unwrap().push("terminate_intent");
6399            if self.intent_failure.load(Ordering::SeqCst) {
6400                return Err(FailureReason::Other(
6401                    "terminate intent directory sync failed".into(),
6402                ));
6403            }
6404            self.intent.store(true, Ordering::SeqCst);
6405            Ok(())
6406        }
6407
6408        fn has_terminate_intent(&self, _attempt: &RunnerAttempt) -> bool {
6409            self.intent.load(Ordering::SeqCst)
6410        }
6411
6412        fn terminate(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6413            assert!(
6414                self.intent.load(Ordering::SeqCst),
6415                "the durable intent must exist before signalling"
6416            );
6417            self.actions.lock().unwrap().push("terminate");
6418            self.terminations.fetch_add(1, Ordering::SeqCst);
6419            self.alive.store(false, Ordering::SeqCst);
6420            Ok(())
6421        }
6422    }
6423
6424    #[derive(Debug, Default)]
6425    struct FakeIsolatedProvider {
6426        resources: Mutex<BTreeMap<AttemptId, (EnvironmentIdentity, EnvironmentState)>>,
6427        extra_resources: Mutex<Vec<EnvironmentIdentity>>,
6428        unsupported_host: AtomicBool,
6429        permission_denied: AtomicBool,
6430        incompatible_image: AtomicBool,
6431        prepare_failure: AtomicBool,
6432        recovery_unavailable: AtomicBool,
6433        destroy_deferred: AtomicBool,
6434        handoffs: AtomicUsize,
6435        completed_successfully: AtomicBool,
6436        native_calls: AtomicUsize,
6437        actions: Mutex<Vec<&'static str>>,
6438    }
6439
6440    impl FakeIsolatedProvider {
6441        fn identity(attempt: &RunnerAttempt, host: HostId) -> EnvironmentIdentity {
6442            let AttemptExecution::Isolated {
6443                provider_kind,
6444                resolved_image,
6445                generation,
6446                ..
6447            } = attempt.execution()
6448            else {
6449                panic!("isolated intent required")
6450            };
6451            EnvironmentIdentity::Isolated {
6452                host,
6453                attempt: attempt.id,
6454                provider_kind: *provider_kind,
6455                environment_id: format!("environment-{}", attempt.id),
6456                resolved_image: resolved_image.clone(),
6457                generation: generation.clone(),
6458            }
6459        }
6460
6461        fn resource_count(&self) -> usize {
6462            self.resources.lock().unwrap().len()
6463        }
6464    }
6465
6466    impl ExecutionProvider for FakeIsolatedProvider {
6467        fn probe(&self, policy: &ScalePolicy) -> ProviderCapability {
6468            if self.unsupported_host.load(Ordering::SeqCst) {
6469                ProviderCapability::Unsupported
6470            } else if self.permission_denied.load(Ordering::SeqCst) {
6471                ProviderCapability::PermissionDenied
6472            } else if policy.execution_policy().is_native() {
6473                ProviderCapability::Unsupported
6474            } else {
6475                ProviderCapability::Ready
6476            }
6477        }
6478
6479        fn resolve(
6480            &self,
6481            policy: &ScalePolicy,
6482        ) -> Result<Option<ResolvedEnvironment>, FailureReason> {
6483            let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
6484                policy.execution_policy()
6485            else {
6486                return Err(FailureReason::Other("isolated policy required".into()));
6487            };
6488            Ok(Some(ResolvedEnvironment {
6489                provider_kind: Backend::Oci,
6490                image: if self.incompatible_image.load(Ordering::SeqCst) {
6491                    ImageReference::new("vm-version:wrong-provider")
6492                        .expect("valid but incompatible")
6493                } else {
6494                    image.clone()
6495                },
6496            }))
6497        }
6498
6499        fn prepare(
6500            &self,
6501            attempt: &RunnerAttempt,
6502            policy: &ScalePolicy,
6503        ) -> Result<PreparedEnvironment, FailureReason> {
6504            assert_eq!(
6505                attempt.state(),
6506                AttemptState::Preparing,
6507                "intent must be durable before provider effect"
6508            );
6509            self.actions.lock().unwrap().push("prepare");
6510            let identity = Self::identity(attempt, policy.host_id);
6511            self.resources
6512                .lock()
6513                .unwrap()
6514                .insert(attempt.id, (identity.clone(), EnvironmentState::Starting));
6515            if self.prepare_failure.swap(false, Ordering::SeqCst) {
6516                return Err(FailureReason::Other("provider-secret-needle".into()));
6517            }
6518            Ok(PreparedEnvironment::isolated(attempt.id, identity))
6519        }
6520
6521        fn start(
6522            &self,
6523            prepared: PreparedEnvironment,
6524            attempt: &RunnerAttempt,
6525            handoff: OneTimeJitHandoff<'_>,
6526        ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
6527            assert_eq!(prepared.attempt, attempt.id);
6528            assert_eq!(attempt.state(), AttemptState::JitReceived);
6529            assert_eq!(handoff.consume().expose(), JIT);
6530            self.handoffs.fetch_add(1, Ordering::SeqCst);
6531            self.actions.lock().unwrap().push("start");
6532            let mut resources = self.resources.lock().unwrap();
6533            let (identity, state) = resources.get_mut(&attempt.id).expect("prepared resource");
6534            *state = EnvironmentState::Running;
6535            Ok(identity.clone())
6536        }
6537
6538        fn inspect(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
6539            self.actions.lock().unwrap().push("inspect");
6540            Ok(self
6541                .resources
6542                .lock()
6543                .unwrap()
6544                .get(&attempt.id)
6545                .map_or(EnvironmentState::Missing, |(_, state)| *state))
6546        }
6547
6548        fn recover(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
6549            if self.recovery_unavailable.load(Ordering::SeqCst) {
6550                return Err(FailureReason::Other(
6551                    "provider discovery unavailable".into(),
6552                ));
6553            }
6554            self.inspect(attempt)
6555        }
6556
6557        fn owns(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
6558            Ok(self
6559                .resources
6560                .lock()
6561                .unwrap()
6562                .get(&attempt.id)
6563                .is_some_and(|(identity, _)| {
6564                    identity_matches_intent(attempt_host(identity), attempt, identity)
6565                }))
6566        }
6567
6568        fn stop(&self, attempt: &RunnerAttempt) -> Result<ProviderStop, FailureReason> {
6569            assert!(self.owns(attempt)?);
6570            self.actions.lock().unwrap().push("stop");
6571            if let Some((_, state)) = self.resources.lock().unwrap().get_mut(&attempt.id) {
6572                *state = EnvironmentState::Exited;
6573            }
6574            Ok(ProviderStop::Stopped)
6575        }
6576
6577        fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
6578            assert!(self.owns(attempt)?);
6579            self.actions.lock().unwrap().push("destroy");
6580            if self.destroy_deferred.load(Ordering::SeqCst) {
6581                return Ok(ProviderDestroy::Deferred("provider busy"));
6582            }
6583            self.resources.lock().unwrap().remove(&attempt.id);
6584            Ok(ProviderDestroy::Destroyed)
6585        }
6586
6587        fn enumerate_owned(&self, host: HostId) -> Vec<EnvironmentIdentity> {
6588            let mut owned = self
6589                .resources
6590                .lock()
6591                .unwrap()
6592                .values()
6593                .filter(|(identity, _)| attempt_host(identity) == host)
6594                .map(|(identity, _)| identity.clone())
6595                .collect::<Vec<_>>();
6596            owned.extend(
6597                self.extra_resources
6598                    .lock()
6599                    .unwrap()
6600                    .iter()
6601                    .filter(|identity| attempt_host(identity) == host)
6602                    .cloned(),
6603            );
6604            owned
6605        }
6606
6607        fn diagnostics(&self, _attempt: &RunnerAttempt) -> Vec<ProviderDiagnostic> {
6608            vec![ProviderDiagnostic::CleanupDeferred; 40]
6609        }
6610
6611        fn spawn(
6612            &self,
6613            _attempt: &RunnerAttempt,
6614            _config: &EncodedJitConfig,
6615        ) -> Result<u32, ProcessStartFailure> {
6616            self.native_calls.fetch_add(1, Ordering::SeqCst);
6617            panic!("isolated provider must never spawn natively")
6618        }
6619        fn is_alive(&self, _attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
6620            panic!("native fallback")
6621        }
6622        fn recovered_pid(&self, _attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
6623            panic!("native fallback")
6624        }
6625        fn completed_successfully(&self, _attempt: &RunnerAttempt) -> bool {
6626            self.completed_successfully.load(Ordering::SeqCst)
6627        }
6628        fn record_terminate_intent(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6629            panic!("native fallback")
6630        }
6631        fn has_terminate_intent(&self, _attempt: &RunnerAttempt) -> bool {
6632            false
6633        }
6634        fn terminate(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6635            panic!("native fallback")
6636        }
6637    }
6638
6639    fn attempt_host(identity: &EnvironmentIdentity) -> HostId {
6640        let EnvironmentIdentity::Isolated { host, .. } = identity else {
6641            panic!("isolated identity")
6642        };
6643        *host
6644    }
6645
6646    #[derive(Debug, Default)]
6647    struct FakeDemand {
6648        answers: Mutex<VecDeque<bool>>,
6649    }
6650
6651    impl FakeDemand {
6652        fn answering(answers: impl IntoIterator<Item = bool>) -> Self {
6653            Self {
6654                answers: Mutex::new(answers.into_iter().collect()),
6655            }
6656        }
6657    }
6658
6659    #[async_trait]
6660    impl DemandPersistence for FakeDemand {
6661        async fn persists(&self, _policy: PolicyId) -> bool {
6662            self.answers.lock().unwrap().pop_front().unwrap_or(true)
6663        }
6664    }
6665
6666    #[derive(Debug, Default)]
6667    struct FakeDelay(Mutex<Vec<Duration>>);
6668
6669    #[async_trait]
6670    impl RetryDelay for FakeDelay {
6671        async fn wait(&self, duration: Duration) {
6672            self.0.lock().unwrap().push(duration);
6673        }
6674    }
6675
6676    #[tokio::test]
6677    async fn live_capacity_reads_the_updated_sqlite_host_and_rejects_a_missing_host() {
6678        let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
6679        let path = harness._root.path().join("live-capacity.sqlite3");
6680        harness.store = Arc::new(SqliteStore::open(&path).unwrap());
6681        harness.store.put_host(&harness.host).unwrap();
6682        harness.launcher = harness.restart();
6683        assert_eq!(
6684            harness.launcher.host_capacity(&harness.host).await.unwrap(),
6685            harness.host.host_capacity
6686        );
6687        // A separate CLI/TUI connection updates the journal while the existing
6688        // launcher keeps its original store connection and startup host.
6689        let writer = SqliteStore::open(&path).unwrap();
6690        let mut updated = harness.host.clone();
6691        updated.host_capacity = std::num::NonZeroU16::new(12).unwrap();
6692        writer.put_host(&updated).unwrap();
6693        assert_eq!(
6694            harness
6695                .launcher
6696                .host_capacity(&harness.host)
6697                .await
6698                .unwrap()
6699                .get(),
6700            12
6701        );
6702
6703        updated.host_capacity = std::num::NonZeroU16::new(2).unwrap();
6704        writer.put_host(&updated).unwrap();
6705        assert_eq!(
6706            harness
6707                .launcher
6708                .host_capacity(&harness.host)
6709                .await
6710                .unwrap()
6711                .get(),
6712            2
6713        );
6714
6715        let mut missing = harness.host.clone();
6716        missing.id = HostId::from_u128(999);
6717        assert_eq!(
6718            harness.launcher.host_capacity(&missing).await,
6719            Err(crate::reconcile::HostCapacityError::MissingHost)
6720        );
6721    }
6722
6723    struct Harness {
6724        _root: tempfile::TempDir,
6725        app_paths: runner_manager_platform::paths::AppPaths,
6726        launcher: LifecycleLauncher,
6727        demand: Arc<dyn DemandPersistence>,
6728        store: Arc<SqliteStore>,
6729        github: Arc<FakeGithubLifecycle>,
6730        packages: Arc<FakePackages>,
6731        processes: Arc<FakeProcesses>,
6732        clock: Arc<FakeClock>,
6733        events: Arc<AttemptEventLog>,
6734        reconcile_events: Arc<crate::reconcile::EventLog>,
6735        delay: Arc<FakeDelay>,
6736        host: runner_manager_domain::model::Host,
6737        policy: ScalePolicy,
6738        allocation_lock: InProcessAllocationLock,
6739        /// The repository persistent root, once one is configured.
6740        workspace_root: Option<LocalAbsolutePath>,
6741    }
6742
6743    impl Harness {
6744        fn new(github: FakeGithubLifecycle, demand: Arc<dyn DemandPersistence>) -> Self {
6745            let root = tempfile::tempdir().unwrap();
6746            let paths = runner_manager_platform::paths::AppPaths::rooted_at(root.path());
6747            paths.create_all().unwrap();
6748            let policy = fixtures::policy()
6749                .repository("octo/repo")
6750                .autoscale("home", 2)
6751                .active()
6752                .build();
6753            // A unit harness must never inspect or write the workstation's
6754            // production runner root. On Windows that is C:\rman, which is
6755            // intentionally inaccessible to an ordinary account after a boot
6756            // service is installed. Using it made the same suite pass on a
6757            // clean CI image and fail on a correctly secured developer host.
6758            let host_root = root.path().join("host-root");
6759            fs::create_dir_all(&host_root).unwrap();
6760            let mut host = fixtures::host().build();
6761            host.runner_root_override = Some(
6762                LocalAbsolutePath::new(host_root.to_str().expect("a UTF-8 temporary path"))
6763                    .expect("a local absolute host root"),
6764            );
6765            let store = Arc::new(SqliteStore::open_in_memory().unwrap());
6766            store.put_host(&host).unwrap();
6767            let github = Arc::new(github);
6768            let packages = Arc::new(FakePackages::default());
6769            let processes = Arc::new(FakeProcesses::default());
6770            let clock = Arc::new(FakeClock::default());
6771            let events = Arc::new(AttemptEventLog::default());
6772            let reconcile_events = Arc::new(crate::reconcile::EventLog::new());
6773            let delay = Arc::new(FakeDelay::default());
6774            let ports = LifecyclePorts {
6775                store: Arc::clone(&store) as Arc<dyn Store>,
6776                github: Arc::clone(&github) as Arc<dyn LifecycleGithub>,
6777                packages: Arc::clone(&packages) as Arc<dyn RuntimePackages>,
6778                processes: Arc::clone(&processes) as Arc<dyn ProcessSupervisor>,
6779                clock: Arc::clone(&clock) as Arc<dyn Clock>,
6780                demand: Arc::clone(&demand),
6781                delay: Arc::clone(&delay) as Arc<dyn RetryDelay>,
6782                events: Arc::clone(&events) as Arc<dyn AttemptEventSink>,
6783                reconcile_events: Arc::clone(&reconcile_events) as Arc<dyn EventSink>,
6784            };
6785            let launcher = Self::launcher_over(policy.host_id, &paths, ports);
6786            Self {
6787                _root: root,
6788                app_paths: paths,
6789                launcher,
6790                demand,
6791                store,
6792                github,
6793                packages,
6794                processes,
6795                clock,
6796                events,
6797                reconcile_events,
6798                delay,
6799                host,
6800                policy,
6801                allocation_lock: InProcessAllocationLock::new(),
6802                workspace_root: None,
6803            }
6804        }
6805
6806        /// Put disposable attempts under a host root of this harness's own.
6807        ///
6808        /// Without it the launcher resolves the *platform* default, which on
6809        /// Windows is `%SystemDrive%\rman` — a real directory on the machine
6810        /// running the suite. Every test added by `c2` places its files inside
6811        /// its own temporary directory instead.
6812        fn with_host_runner_root(mut self) -> Self {
6813            let host_root = self.host_root();
6814            fs::create_dir_all(&host_root).unwrap();
6815            self.host.runner_root_override = Some(
6816                LocalAbsolutePath::new(host_root.to_str().expect("a UTF-8 temporary path"))
6817                    .expect("a local absolute host root"),
6818            );
6819            self.store.put_host(&self.host).unwrap();
6820            self
6821        }
6822
6823        /// Opt this harness's repository into a persistent workspace (D4).
6824        fn with_persistent_workspace(mut self, capacity: u16) -> Self {
6825            self = self.with_host_runner_root();
6826            let root = self._root.path().join("persist");
6827            let root = LocalAbsolutePath::new(root.to_str().expect("a UTF-8 temporary path"))
6828                .expect("a local absolute workspace root");
6829            self.policy = fixtures::policy()
6830                .repository("octo/repo")
6831                .autoscale("home", capacity)
6832                .active()
6833                .build();
6834            self.policy
6835                .set_workspace_policy(
6836                    WorkspacePolicy::persistent(root.clone(), TargetScope::Repository)
6837                        .expect("a repository may be persistent"),
6838                )
6839                .expect("a repository may be persistent");
6840            self.workspace_root = Some(root);
6841            // The journal's copy, so that cleanup's cross-check against a
6842            // surviving policy is exercised rather than skipped.
6843            self.store.insert_policy(&self.policy).unwrap();
6844            self
6845        }
6846
6847        fn workspace_root(&self) -> &LocalAbsolutePath {
6848            self.workspace_root
6849                .as_ref()
6850                .expect("this harness configured a persistent workspace")
6851        }
6852
6853        fn slot_path(&self, slot: u16) -> PathBuf {
6854            self.workspace_root().as_path().join(format!("s{slot}"))
6855        }
6856
6857        fn host_root(&self) -> PathBuf {
6858            self._root.path().join("host-root")
6859        }
6860
6861        fn attempt(&self, id: AttemptId) -> RunnerAttempt {
6862            self.store
6863                .attempt(id)
6864                .unwrap()
6865                .expect("the attempt is journalled")
6866        }
6867
6868        /// Conclude an attempt and run the real cleanup over it.
6869        fn conclude(&self, id: AttemptId) -> RunnerAttempt {
6870            let mut attempt = self.attempt(id);
6871            attempt
6872                .conclude(
6873                    AttemptOutcome::failed(FailureReason::ProcessExitedUnexpectedly),
6874                    self.clock.now(),
6875                )
6876                .unwrap();
6877            self.store.record_attempt(&attempt).unwrap();
6878            attempt
6879        }
6880
6881        async fn cleanup_retaining_work(&self, id: AttemptId) {
6882            self.conclude(id);
6883            self.launcher
6884                .clean(id)
6885                .await
6886                .expect("the slot is scrubbed and the lease released");
6887        }
6888
6889        /// The launcher configuration every launcher in this harness shares, so
6890        /// that the one a restart mints cannot drift from the original.
6891        fn launcher_over(
6892            host: HostId,
6893            paths: &runner_manager_platform::paths::AppPaths,
6894            ports: LifecyclePorts,
6895        ) -> LifecycleLauncher {
6896            LifecycleLauncher::new(
6897                host,
6898                paths.clone(),
6899                paths.logs_dir(),
6900                1,
6901                RecoveryTimeouts::new(
6902                    Elapsed::seconds(10),
6903                    Elapsed::seconds(10),
6904                    Elapsed::seconds(10),
6905                ),
6906                RetryPolicy::bounded(3, Duration::from_millis(10), Duration::from_millis(25)),
6907                ports,
6908            )
6909        }
6910
6911        /// The same journal, the same directories, a launcher that remembers
6912        /// nothing — which is what a daemon restart is.
6913        fn restart(&self) -> LifecycleLauncher {
6914            Self::launcher_over(
6915                self.policy.host_id,
6916                &self.app_paths,
6917                LifecyclePorts {
6918                    store: Arc::clone(&self.store) as Arc<dyn Store>,
6919                    github: Arc::clone(&self.github) as Arc<dyn LifecycleGithub>,
6920                    packages: Arc::clone(&self.packages) as Arc<dyn RuntimePackages>,
6921                    processes: Arc::clone(&self.processes) as Arc<dyn ProcessSupervisor>,
6922                    clock: Arc::clone(&self.clock) as Arc<dyn Clock>,
6923                    demand: Arc::clone(&self.demand),
6924                    delay: Arc::clone(&self.delay) as Arc<dyn RetryDelay>,
6925                    events: Arc::clone(&self.events) as Arc<dyn AttemptEventSink>,
6926                    reconcile_events: Arc::clone(&self.reconcile_events) as Arc<dyn EventSink>,
6927                },
6928            )
6929        }
6930
6931        async fn ready(&self) {
6932            self.launcher
6933                .recover_startup(std::slice::from_ref(&self.policy))
6934                .await
6935                .unwrap();
6936        }
6937
6938        async fn launch(&self) -> RunnerAttempt {
6939            self.launch_result().await.unwrap()
6940        }
6941
6942        async fn launch_result(&self) -> Result<RunnerAttempt, LaunchFailure> {
6943            self.launch_after_waiting(Duration::ZERO).await
6944        }
6945
6946        async fn launch_after_waiting(
6947            &self,
6948            lock_wait: Duration,
6949        ) -> Result<RunnerAttempt, LaunchFailure> {
6950            let reserved = self.reserve().await?;
6951            self.launcher
6952                .finish_launch(
6953                    &self.policy,
6954                    reserved,
6955                    ReservationWaits {
6956                        lock: lock_wait,
6957                        order: Duration::from_secs(2),
6958                    },
6959                )
6960                .await
6961        }
6962
6963        /// A whole launch through `launcher`, as a reconciliation pass makes it.
6964        async fn launch_through(
6965            &self,
6966            launcher: &LifecycleLauncher,
6967        ) -> Result<RunnerAttempt, LaunchFailure> {
6968            let reserved = self.reserve_through(launcher).await?;
6969            launcher
6970                .finish_launch(&self.policy, reserved, ReservationWaits::default())
6971                .await
6972        }
6973
6974        async fn reserve(&self) -> Result<RunnerAttempt, LaunchFailure> {
6975            self.reserve_through(&self.launcher).await
6976        }
6977
6978        async fn reserve_through(
6979            &self,
6980            launcher: &LifecycleLauncher,
6981        ) -> Result<RunnerAttempt, LaunchFailure> {
6982            let guard = self.allocation_lock.acquire().await.unwrap();
6983            launcher
6984                .launch(LaunchRequest {
6985                    host: &self.host,
6986                    policy: &self.policy,
6987                    allocation_guard: &guard,
6988                })
6989                .await
6990        }
6991
6992        fn only_attempt(&self) -> RunnerAttempt {
6993            self.store.attempts().unwrap().into_iter().next().unwrap()
6994        }
6995    }
6996
6997    fn isolated_harness() -> Harness {
6998        use runner_manager_domain::execution::{ExecutionPolicy, ResourceLimits};
6999        let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
7000            .with_host_runner_root();
7001        let image =
7002            ImageReference::new(format!("registry.example/runner@sha256:{}", "a".repeat(64)))
7003                .expect("pinned image");
7004        harness
7005            .policy
7006            .set_execution_policy(ExecutionPolicy::Isolated {
7007                backend: Backend::Oci,
7008                image,
7009                resources: ResourceLimits {
7010                    cpu_millis: 1000,
7011                    memory_mib: 1024,
7012                    disk_mib: 4096,
7013                },
7014            })
7015            .expect("isolated policy");
7016        harness.github.watch_journal(Arc::clone(&harness.store));
7017        harness
7018    }
7019
7020    fn launcher_with_isolated(
7021        harness: &Harness,
7022        provider: Arc<FakeIsolatedProvider>,
7023    ) -> LifecycleLauncher {
7024        Harness::launcher_over(
7025            harness.policy.host_id,
7026            &harness.app_paths,
7027            LifecyclePorts {
7028                store: Arc::clone(&harness.store) as Arc<dyn Store>,
7029                github: Arc::clone(&harness.github) as Arc<dyn LifecycleGithub>,
7030                packages: Arc::clone(&harness.packages) as Arc<dyn RuntimePackages>,
7031                processes: provider as Arc<dyn ExecutionProvider>,
7032                clock: Arc::clone(&harness.clock) as Arc<dyn Clock>,
7033                demand: Arc::clone(&harness.demand),
7034                delay: Arc::clone(&harness.delay) as Arc<dyn RetryDelay>,
7035                events: Arc::clone(&harness.events) as Arc<dyn AttemptEventSink>,
7036                reconcile_events: Arc::clone(&harness.reconcile_events) as Arc<dyn EventSink>,
7037            },
7038        )
7039    }
7040
7041    #[tokio::test]
7042    async fn isolated_policy_and_journal_never_enter_native_provider_or_jit() {
7043        use runner_manager_domain::execution::{
7044            AttemptExecution, Backend, ExecutionPolicy, ImageReference, ResourceLimits,
7045        };
7046
7047        let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
7048            .with_host_runner_root();
7049        let image =
7050            ImageReference::new(format!("registry.example/runner@sha256:{}", "a".repeat(64)))
7051                .expect("pinned image");
7052        harness
7053            .policy
7054            .set_execution_policy(ExecutionPolicy::Isolated {
7055                backend: Backend::Oci,
7056                image: image.clone(),
7057                resources: ResourceLimits {
7058                    cpu_millis: 1000,
7059                    memory_mib: 1024,
7060                    disk_mib: 4096,
7061                },
7062            })
7063            .expect("valid isolated profile");
7064
7065        harness.ready().await;
7066        assert_eq!(
7067            harness.processes.probe(&harness.policy),
7068            ProviderCapability::Unsupported
7069        );
7070        let failure = harness
7071            .launch_result()
7072            .await
7073            .expect_err("provider is unavailable");
7074        assert_eq!(
7075            failure.reason,
7076            FailureReason::IsolationProvider(IsolationProviderFailure::Unsupported)
7077        );
7078
7079        let runtime = harness.host_root().join("isolated-owned-environment");
7080        fs::create_dir_all(&runtime).expect("provider resource fixture");
7081        fs::write(runtime.join("owned-marker"), "non-secret").expect("fixture marker");
7082        let mut attempt = RunnerAttempt::allocate(
7083            AttemptId::from_u128(0x123),
7084            harness.policy.id,
7085            runtime.clone(),
7086            harness.clock.now(),
7087        );
7088        attempt
7089            .allocate_execution(AttemptExecution::Isolated {
7090                provider_kind: Backend::Oci,
7091                environment_id: None,
7092                resolved_image: image,
7093                generation: "generation-123".into(),
7094            })
7095            .expect("isolated intent");
7096        attempt
7097            .prepared_environment("environment-123".into())
7098            .expect("prepared identity");
7099        harness
7100            .store
7101            .record_attempt(&attempt)
7102            .expect("durable isolated journal");
7103
7104        harness
7105            .launcher
7106            .recover_startup(std::slice::from_ref(&harness.policy))
7107            .await
7108            .expect("unsupported isolated cleanup is profile-local");
7109        let failure = harness
7110            .launch_result()
7111            .await
7112            .expect_err("provider unavailable for this policy");
7113        assert_eq!(
7114            failure.reason,
7115            FailureReason::IsolationProvider(IsolationProviderFailure::Unsupported)
7116        );
7117        harness
7118            .launcher
7119            .supervise(&harness.policy)
7120            .await
7121            .expect("unavailable attempt is deferred");
7122        assert!(harness.launcher.clean(attempt.id).await.is_err());
7123        assert_eq!(
7124            harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7125            AttemptState::CleanupDeferred
7126        );
7127        assert!(matches!(
7128            harness.launcher.recover_startup(&[]).await,
7129            Err(LifecycleError::RecoveryIncomplete)
7130        ));
7131        assert!(
7132            runtime.join("owned-marker").is_file(),
7133            "native cleanup touched the isolated environment"
7134        );
7135        assert!(
7136            harness.processes.actions.lock().unwrap().is_empty(),
7137            "native recovery inspected an isolated identity"
7138        );
7139        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
7140        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7141        assert_eq!(harness.packages.materializations.load(Ordering::SeqCst), 0);
7142    }
7143
7144    #[tokio::test]
7145    async fn isolated_prepare_is_durable_before_jit_and_start_uses_one_handoff() {
7146        let harness = isolated_harness();
7147        let provider = Arc::new(FakeIsolatedProvider::default());
7148        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7149        launcher
7150            .recover_startup(std::slice::from_ref(&harness.policy))
7151            .await
7152            .expect("ready");
7153        let attempt = harness
7154            .launch_through(&launcher)
7155            .await
7156            .expect("isolated launch");
7157        assert_eq!(attempt.state(), AttemptState::Starting);
7158        assert_eq!(provider.resource_count(), 1);
7159        assert_eq!(provider.handoffs.load(Ordering::SeqCst), 1);
7160        assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7161        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 1);
7162        let fact = harness
7163            .github
7164            .registration_facts()
7165            .pop()
7166            .expect("JIT registration");
7167        assert_eq!(
7168            fact.journal_states,
7169            vec![AttemptState::Prepared],
7170            "provider readiness preceded GitHub JIT"
7171        );
7172        assert_eq!(
7173            harness
7174                .events
7175                .events()
7176                .into_iter()
7177                .filter_map(|event| match event {
7178                    AttemptEvent::State { state, .. } => Some(state),
7179                    _ => None,
7180                })
7181                .collect::<Vec<_>>(),
7182            vec![
7183                AttemptState::Allocated,
7184                AttemptState::Preparing,
7185                AttemptState::Prepared,
7186                AttemptState::JitReceived,
7187                AttemptState::Starting
7188            ]
7189        );
7190        let restarted = launcher_with_isolated(&harness, Arc::clone(&provider));
7191        restarted
7192            .recover_startup(std::slice::from_ref(&harness.policy))
7193            .await
7194            .expect("adopt live sandbox");
7195        assert_eq!(provider.resource_count(), 1);
7196        assert_eq!(
7197            harness.github.registrations.load(Ordering::SeqCst),
7198            1,
7199            "restart cannot issue duplicate JIT"
7200        );
7201        harness
7202            .github
7203            .observe(GithubRunnerObservation::Registered { busy: true });
7204        restarted
7205            .supervise(&harness.policy)
7206            .await
7207            .expect("job observed");
7208        assert_eq!(
7209            harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7210            AttemptState::Busy
7211        );
7212        provider
7213            .resources
7214            .lock()
7215            .unwrap()
7216            .get_mut(&attempt.id)
7217            .unwrap()
7218            .1 = EnvironmentState::Exited;
7219        harness
7220            .github
7221            .observe(GithubRunnerObservation::NotRegistered);
7222        restarted
7223            .supervise(&harness.policy)
7224            .await
7225            .expect("terminal cleanup");
7226        assert_eq!(provider.resource_count(), 0);
7227        assert_eq!(
7228            harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7229            AttemptState::Cleaned
7230        );
7231    }
7232
7233    #[tokio::test]
7234    async fn attached_isolated_job_exit_is_recorded_as_completed() {
7235        let harness = isolated_harness();
7236        let provider = Arc::new(FakeIsolatedProvider::default());
7237        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7238        launcher
7239            .recover_startup(std::slice::from_ref(&harness.policy))
7240            .await
7241            .unwrap();
7242        let attempt = harness.launch_through(&launcher).await.unwrap();
7243        harness
7244            .github
7245            .observe(GithubRunnerObservation::Registered { busy: true });
7246        launcher.supervise(&harness.policy).await.unwrap();
7247        provider
7248            .resources
7249            .lock()
7250            .unwrap()
7251            .get_mut(&attempt.id)
7252            .unwrap()
7253            .1 = EnvironmentState::Exited;
7254        provider
7255            .completed_successfully
7256            .store(true, Ordering::SeqCst);
7257        harness
7258            .github
7259            .observe(GithubRunnerObservation::NotRegistered);
7260        launcher.supervise(&harness.policy).await.unwrap();
7261        let cleaned = harness.store.attempt(attempt.id).unwrap().unwrap();
7262        assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
7263        assert_eq!(cleaned.state(), AttemptState::Cleaned);
7264    }
7265
7266    #[tokio::test]
7267    async fn isolated_prepare_failure_stays_local_and_crash_gap_recovers_owned_resource() {
7268        let harness = isolated_harness();
7269        let provider = Arc::new(FakeIsolatedProvider::default());
7270        provider.prepare_failure.store(true, Ordering::SeqCst);
7271        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7272        launcher
7273            .recover_startup(std::slice::from_ref(&harness.policy))
7274            .await
7275            .expect("ready");
7276        let failure = harness
7277            .launch_through(&launcher)
7278            .await
7279            .expect_err("prepare failed after creating resource");
7280        assert_eq!(
7281            failure.reason,
7282            FailureReason::IsolationProvider(IsolationProviderFailure::RuntimeOperationFailed)
7283        );
7284        assert_eq!(provider.resource_count(), 1);
7285        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7286        assert!(
7287            !harness
7288                .store
7289                .dump_text()
7290                .unwrap()
7291                .contains("provider-secret-needle")
7292        );
7293        let id = harness.store.attempts().unwrap()[0].id;
7294        assert_eq!(
7295            harness.store.attempt(id).unwrap().unwrap().outcome(),
7296            Some(&AttemptOutcome::failed(FailureReason::IsolationProvider(
7297                IsolationProviderFailure::RuntimeOperationFailed
7298            )))
7299        );
7300        let restarted = launcher_with_isolated(&harness, Arc::clone(&provider));
7301        restarted
7302            .recover_startup(std::slice::from_ref(&harness.policy))
7303            .await
7304            .expect("profile-local recovery");
7305        assert_eq!(
7306            provider.resource_count(),
7307            0,
7308            "the exact owned resource was destroyed once"
7309        );
7310        assert_eq!(
7311            harness.store.attempt(id).unwrap().unwrap().state(),
7312            AttemptState::Cleaned
7313        );
7314        let diagnostic =
7315            fs::read_to_string(restarted.diagnostics_root.join(format!("{id}.log"))).unwrap();
7316        assert!(diagnostic.contains("reason=isolation_runtime_operation_failed"));
7317        assert!(!diagnostic.contains("provider-secret-needle"));
7318        assert_eq!(
7319            provider
7320                .actions
7321                .lock()
7322                .unwrap()
7323                .iter()
7324                .filter(|action| **action == "destroy")
7325                .count(),
7326            1
7327        );
7328        assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7329    }
7330
7331    #[test]
7332    fn isolation_capability_and_adapter_failures_keep_closed_categories() {
7333        let cases = [
7334            (
7335                ProviderCapability::NotInstalled,
7336                IsolationProviderFailure::NotInstalled,
7337            ),
7338            (
7339                ProviderCapability::PermissionDenied,
7340                IsolationProviderFailure::PermissionDenied,
7341            ),
7342            (
7343                ProviderCapability::ImageUnavailableOrIncompatible,
7344                IsolationProviderFailure::ImageUnavailableOrIncompatible,
7345            ),
7346            (
7347                ProviderCapability::DiskQuotaUnavailable,
7348                IsolationProviderFailure::DiskQuotaUnavailable,
7349            ),
7350            (
7351                ProviderCapability::Degraded,
7352                IsolationProviderFailure::Degraded,
7353            ),
7354        ];
7355        for (capability, category) in cases {
7356            assert_eq!(
7357                capability.refusal(),
7358                FailureReason::IsolationProvider(category)
7359            );
7360        }
7361        let raw = FailureReason::Other("ghp_secret_from_provider".into());
7362        let safe = safe_isolation_reason(raw, IsolationProviderFailure::RuntimeOperationFailed);
7363        assert_eq!(
7364            safe,
7365            FailureReason::IsolationProvider(IsolationProviderFailure::RuntimeOperationFailed)
7366        );
7367        assert!(!safe.to_string().contains("ghp_secret"));
7368        assert_eq!(
7369            safe_isolation_reason(
7370                FailureReason::IsolationProvider(IsolationProviderFailure::PermissionDenied),
7371                IsolationProviderFailure::RuntimeOperationFailed,
7372            ),
7373            FailureReason::IsolationProvider(IsolationProviderFailure::PermissionDenied)
7374        );
7375    }
7376
7377    #[tokio::test]
7378    async fn isolated_permission_and_image_failures_stop_before_jit_without_native_fallback() {
7379        for image_failure in [false, true] {
7380            let harness = isolated_harness();
7381            let provider = Arc::new(FakeIsolatedProvider::default());
7382            provider
7383                .permission_denied
7384                .store(!image_failure, Ordering::SeqCst);
7385            provider
7386                .incompatible_image
7387                .store(image_failure, Ordering::SeqCst);
7388            let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7389            launcher
7390                .recover_startup(std::slice::from_ref(&harness.policy))
7391                .await
7392                .unwrap();
7393            let error = harness
7394                .launch_through(&launcher)
7395                .await
7396                .expect_err("pre-JIT refusal");
7397            let expected = if image_failure {
7398                "resolved image is incompatible"
7399            } else {
7400                "permission denied"
7401            };
7402            assert!(error.reason.to_string().contains(expected));
7403            assert!(
7404                harness.store.attempts().unwrap().is_empty(),
7405                "no allocation was journalled"
7406            );
7407            assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7408            assert_eq!(provider.handoffs.load(Ordering::SeqCst), 0);
7409            assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7410        }
7411    }
7412
7413    #[tokio::test]
7414    async fn unsupported_host_stops_before_jit_without_allocation_or_native_fallback() {
7415        let harness = isolated_harness();
7416        let provider = Arc::new(FakeIsolatedProvider::default());
7417        provider.unsupported_host.store(true, Ordering::SeqCst);
7418        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7419        launcher
7420            .recover_startup(std::slice::from_ref(&harness.policy))
7421            .await
7422            .unwrap();
7423        let error = harness
7424            .launch_through(&launcher)
7425            .await
7426            .expect_err("unsupported host must be refused before JIT");
7427
7428        assert_eq!(
7429            error.reason,
7430            FailureReason::IsolationProvider(IsolationProviderFailure::Unsupported)
7431        );
7432        assert!(harness.store.attempts().unwrap().is_empty());
7433        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7434        assert_eq!(provider.handoffs.load(Ordering::SeqCst), 0);
7435        assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7436        assert!(provider.actions.lock().unwrap().is_empty());
7437    }
7438
7439    #[tokio::test]
7440    async fn no_journalled_environment_id_waits_for_provider_absence_proof() {
7441        let harness = isolated_harness();
7442        let provider = Arc::new(FakeIsolatedProvider::default());
7443        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7444        let runtime = harness.host_root().join("prepare-without-identity");
7445        fs::create_dir_all(&runtime).unwrap();
7446        let mut attempt = RunnerAttempt::allocate(
7447            AttemptId::from_u128(0x9191),
7448            harness.policy.id,
7449            runtime,
7450            harness.clock.now(),
7451        );
7452        let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7453            harness.policy.execution_policy()
7454        else {
7455            unreachable!()
7456        };
7457        attempt
7458            .allocate_execution(AttemptExecution::Isolated {
7459                provider_kind: Backend::Oci,
7460                environment_id: None,
7461                resolved_image: image.clone(),
7462                generation: "prepare-generation".into(),
7463            })
7464            .unwrap();
7465        attempt.begin_prepare(harness.clock.now()).unwrap();
7466        attempt
7467            .conclude(
7468                AttemptOutcome::failed(FailureReason::Other("prepare interrupted".into())),
7469                harness.clock.now(),
7470            )
7471            .unwrap();
7472        harness.store.record_attempt(&attempt).unwrap();
7473
7474        provider.recovery_unavailable.store(true, Ordering::SeqCst);
7475        launcher
7476            .clean(attempt.id)
7477            .await
7478            .expect_err("absence unproven");
7479        let deferred = harness.store.attempt(attempt.id).unwrap().unwrap();
7480        assert_eq!(deferred.state(), AttemptState::CleanupDeferred);
7481        assert!(deferred.counts_against_capacity());
7482
7483        provider.recovery_unavailable.store(false, Ordering::SeqCst);
7484        launcher.clean(attempt.id).await.expect("absence proven");
7485        assert_eq!(
7486            harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7487            AttemptState::Cleaned
7488        );
7489    }
7490
7491    #[tokio::test]
7492    async fn isolated_terminal_capacity_waits_for_confirmed_absence_and_generation_matches() {
7493        let harness = isolated_harness();
7494        let provider = Arc::new(FakeIsolatedProvider::default());
7495        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7496        launcher
7497            .recover_startup(std::slice::from_ref(&harness.policy))
7498            .await
7499            .expect("ready");
7500        let mut attempt = harness.launch_through(&launcher).await.expect("launch");
7501        attempt
7502            .conclude(
7503                AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7504                harness.clock.now(),
7505            )
7506            .unwrap();
7507        harness.store.record_attempt(&attempt).unwrap();
7508        provider.destroy_deferred.store(true, Ordering::SeqCst);
7509        launcher
7510            .clean(attempt.id)
7511            .await
7512            .expect_err("provider deferred destroy");
7513        let deferred = harness.store.attempt(attempt.id).unwrap().unwrap();
7514        assert_eq!(deferred.state(), AttemptState::CleanupDeferred);
7515        assert!(deferred.counts_against_capacity());
7516        assert_eq!(provider.resource_count(), 1);
7517        provider.destroy_deferred.store(false, Ordering::SeqCst);
7518        let restarted = launcher_with_isolated(&harness, Arc::clone(&provider));
7519        restarted
7520            .recover_startup(std::slice::from_ref(&harness.policy))
7521            .await
7522            .expect("retry cleanup");
7523        assert_eq!(provider.resource_count(), 0);
7524        assert!(
7525            !harness
7526                .store
7527                .attempt(attempt.id)
7528                .unwrap()
7529                .unwrap()
7530                .counts_against_capacity()
7531        );
7532
7533        // A reused runtime name with a different generation is not ours to delete.
7534        let second_id = AttemptId::from_u128(0x9999);
7535        let second_runtime = harness.host_root().join("generation-mismatch");
7536        fs::create_dir_all(&second_runtime).unwrap();
7537        let mut terminal = RunnerAttempt::allocate(
7538            second_id,
7539            harness.policy.id,
7540            second_runtime,
7541            harness.clock.now(),
7542        );
7543        let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7544            harness.policy.execution_policy()
7545        else {
7546            unreachable!()
7547        };
7548        terminal
7549            .allocate_execution(AttemptExecution::Isolated {
7550                provider_kind: Backend::Oci,
7551                environment_id: None,
7552                resolved_image: image.clone(),
7553                generation: "expected-generation".into(),
7554            })
7555            .unwrap();
7556        let mut stranger = FakeIsolatedProvider::identity(&terminal, harness.policy.host_id);
7557        let EnvironmentIdentity::Isolated { environment_id, .. } = &stranger else {
7558            unreachable!()
7559        };
7560        terminal.begin_prepare(harness.clock.now()).unwrap();
7561        terminal
7562            .prepared_environment(environment_id.clone())
7563            .unwrap();
7564        terminal.mark_prepared(harness.clock.now()).unwrap();
7565        terminal.jit_received(harness.clock.now()).unwrap();
7566        terminal.started_isolated(harness.clock.now()).unwrap();
7567        terminal
7568            .conclude(
7569                AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7570                harness.clock.now(),
7571            )
7572            .unwrap();
7573        harness.store.record_attempt(&terminal).unwrap();
7574        let EnvironmentIdentity::Isolated { generation, .. } = &mut stranger else {
7575            unreachable!()
7576        };
7577        *generation = "another-generation".into();
7578        provider
7579            .resources
7580            .lock()
7581            .unwrap()
7582            .insert(second_id, (stranger, EnvironmentState::Running));
7583        restarted
7584            .clean(terminal.id)
7585            .await
7586            .expect_err("ownership mismatch quarantined");
7587        assert_eq!(provider.resource_count(), 1);
7588        assert_eq!(
7589            harness.store.attempt(terminal.id).unwrap().unwrap().state(),
7590            AttemptState::CleanupDeferred
7591        );
7592        assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7593    }
7594
7595    #[tokio::test]
7596    async fn isolated_restart_at_each_transition_adopts_or_destroys_one_resource() {
7597        for stage in [
7598            AttemptState::Allocated,
7599            AttemptState::Preparing,
7600            AttemptState::Prepared,
7601            AttemptState::JitReceived,
7602            AttemptState::Starting,
7603            AttemptState::Idle,
7604            AttemptState::Busy,
7605            AttemptState::Finished,
7606            AttemptState::Failed,
7607            AttemptState::Orphaned,
7608            AttemptState::Destroying,
7609            AttemptState::CleanupDeferred,
7610        ] {
7611            let harness = isolated_harness();
7612            let provider = Arc::new(FakeIsolatedProvider::default());
7613            let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7614            let id = AttemptId::from_u128(0x1000 + stage as u128);
7615            let runtime = harness.host_root().join(format!("phase-{stage}"));
7616            fs::create_dir_all(&runtime).unwrap();
7617            let mut attempt =
7618                RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
7619            let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7620                harness.policy.execution_policy()
7621            else {
7622                unreachable!()
7623            };
7624            attempt
7625                .allocate_execution(AttemptExecution::Isolated {
7626                    provider_kind: Backend::Oci,
7627                    environment_id: None,
7628                    resolved_image: image.clone(),
7629                    generation: format!("generation-{stage}"),
7630                })
7631                .unwrap();
7632            if stage != AttemptState::Allocated {
7633                attempt.begin_prepare(harness.clock.now()).unwrap();
7634            }
7635            let identity = FakeIsolatedProvider::identity(&attempt, harness.policy.host_id);
7636            if stage != AttemptState::Allocated {
7637                let resource_state = if matches!(
7638                    stage,
7639                    AttemptState::Finished
7640                        | AttemptState::Failed
7641                        | AttemptState::Orphaned
7642                        | AttemptState::Destroying
7643                        | AttemptState::CleanupDeferred
7644                ) {
7645                    EnvironmentState::Exited
7646                } else {
7647                    EnvironmentState::Running
7648                };
7649                provider
7650                    .resources
7651                    .lock()
7652                    .unwrap()
7653                    .insert(id, (identity.clone(), resource_state));
7654            }
7655            if !matches!(stage, AttemptState::Allocated | AttemptState::Preparing) {
7656                let EnvironmentIdentity::Isolated { environment_id, .. } = &identity else {
7657                    unreachable!()
7658                };
7659                attempt
7660                    .prepared_environment(environment_id.clone())
7661                    .unwrap();
7662                attempt.mark_prepared(harness.clock.now()).unwrap();
7663            }
7664            if matches!(
7665                stage,
7666                AttemptState::JitReceived
7667                    | AttemptState::Starting
7668                    | AttemptState::Idle
7669                    | AttemptState::Busy
7670                    | AttemptState::Finished
7671                    | AttemptState::Failed
7672                    | AttemptState::Orphaned
7673                    | AttemptState::Destroying
7674                    | AttemptState::CleanupDeferred
7675            ) {
7676                attempt.jit_received(harness.clock.now()).unwrap();
7677            }
7678            if matches!(
7679                stage,
7680                AttemptState::Starting
7681                    | AttemptState::Idle
7682                    | AttemptState::Busy
7683                    | AttemptState::Finished
7684                    | AttemptState::Failed
7685                    | AttemptState::Orphaned
7686                    | AttemptState::Destroying
7687                    | AttemptState::CleanupDeferred
7688            ) {
7689                attempt.started_isolated(harness.clock.now()).unwrap();
7690            }
7691            if matches!(
7692                stage,
7693                AttemptState::Idle | AttemptState::Busy | AttemptState::Finished
7694            ) {
7695                attempt.registered_idle(73, harness.clock.now()).unwrap();
7696            }
7697            if matches!(stage, AttemptState::Busy | AttemptState::Finished) {
7698                attempt.assigned_job(73, harness.clock.now()).unwrap();
7699            }
7700            if matches!(
7701                stage,
7702                AttemptState::Finished
7703                    | AttemptState::Failed
7704                    | AttemptState::Orphaned
7705                    | AttemptState::Destroying
7706                    | AttemptState::CleanupDeferred
7707            ) {
7708                let outcome = match stage {
7709                    AttemptState::Finished => AttemptOutcome::CompletedJob,
7710                    AttemptState::Orphaned => AttemptOutcome::Orphaned,
7711                    _ => AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7712                };
7713                attempt.conclude(outcome, harness.clock.now()).unwrap();
7714            }
7715            if matches!(
7716                stage,
7717                AttemptState::Destroying | AttemptState::CleanupDeferred
7718            ) {
7719                attempt.begin_destroy(harness.clock.now()).unwrap();
7720            }
7721            if stage == AttemptState::CleanupDeferred {
7722                attempt.defer_cleanup(harness.clock.now()).unwrap();
7723            }
7724            harness.store.record_attempt(&attempt).unwrap();
7725            launcher
7726                .recover_startup(std::slice::from_ref(&harness.policy))
7727                .await
7728                .unwrap_or_else(|error| panic!("restart from {stage}: {error}"));
7729            let resumed = harness.store.attempt(id).unwrap().unwrap();
7730            if matches!(
7731                stage,
7732                AttemptState::Allocated
7733                    | AttemptState::Preparing
7734                    | AttemptState::Prepared
7735                    | AttemptState::Finished
7736                    | AttemptState::Failed
7737                    | AttemptState::Orphaned
7738                    | AttemptState::Destroying
7739                    | AttemptState::CleanupDeferred
7740            ) {
7741                assert_eq!(
7742                    resumed.state(),
7743                    AttemptState::Cleaned,
7744                    "{stage} must retire after restart"
7745                );
7746            } else {
7747                assert!(resumed.counts_against_capacity(), "live {stage} is adopted");
7748                assert_eq!(provider.resource_count(), 1);
7749                provider.resources.lock().unwrap().get_mut(&id).unwrap().1 =
7750                    EnvironmentState::Exited;
7751                launcher.supervise(&harness.policy).await.unwrap();
7752                assert_eq!(
7753                    harness.store.attempt(id).unwrap().unwrap().state(),
7754                    AttemptState::Cleaned
7755                );
7756            }
7757            assert_eq!(
7758                provider.resource_count(),
7759                0,
7760                "{stage} resource cleaned exactly once"
7761            );
7762            let destroyed = provider
7763                .actions
7764                .lock()
7765                .unwrap()
7766                .iter()
7767                .filter(|action| **action == "destroy")
7768                .count();
7769            assert_eq!(
7770                destroyed,
7771                usize::from(stage != AttemptState::Allocated),
7772                "{stage} destroy count"
7773            );
7774            assert_eq!(
7775                harness.github.registrations.load(Ordering::SeqCst),
7776                0,
7777                "restart from {stage} cannot duplicate JIT"
7778            );
7779            assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7780        }
7781    }
7782
7783    #[tokio::test]
7784    async fn duplicate_provider_ownership_is_quarantined_until_unambiguous() {
7785        let harness = isolated_harness();
7786        let provider = Arc::new(FakeIsolatedProvider::default());
7787        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7788        let id = AttemptId::from_u128(0x7777);
7789        let runtime = harness.host_root().join("duplicate-provider-ownership");
7790        fs::create_dir_all(&runtime).unwrap();
7791        let mut attempt =
7792            RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
7793        let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7794            harness.policy.execution_policy()
7795        else {
7796            unreachable!()
7797        };
7798        attempt
7799            .allocate_execution(AttemptExecution::Isolated {
7800                provider_kind: Backend::Oci,
7801                environment_id: None,
7802                resolved_image: image.clone(),
7803                generation: "duplicate-generation".into(),
7804            })
7805            .unwrap();
7806        attempt.begin_prepare(harness.clock.now()).unwrap();
7807        harness.store.record_attempt(&attempt).unwrap();
7808        let first = FakeIsolatedProvider::identity(&attempt, harness.policy.host_id);
7809        let mut second = first.clone();
7810        let EnvironmentIdentity::Isolated { environment_id, .. } = &mut second else {
7811            unreachable!()
7812        };
7813        *environment_id = "another-environment".into();
7814        provider
7815            .resources
7816            .lock()
7817            .unwrap()
7818            .insert(id, (first, EnvironmentState::Running));
7819        provider.extra_resources.lock().unwrap().push(second);
7820        launcher
7821            .recover_startup(std::slice::from_ref(&harness.policy))
7822            .await
7823            .expect("duplicate is local");
7824        let deferred = harness.store.attempt(id).unwrap().unwrap();
7825        assert_eq!(deferred.state(), AttemptState::CleanupDeferred);
7826        assert!(deferred.counts_against_capacity());
7827        assert_eq!(provider.resource_count(), 1);
7828        assert_eq!(
7829            provider
7830                .actions
7831                .lock()
7832                .unwrap()
7833                .iter()
7834                .filter(|action| **action == "destroy")
7835                .count(),
7836            0
7837        );
7838        provider.extra_resources.lock().unwrap().clear();
7839        // A tolerated cleanup failure is retried after its backoff delay.
7840        harness.clock.advance_secs(31);
7841        launcher
7842            .recover_startup(std::slice::from_ref(&harness.policy))
7843            .await
7844            .expect("single identity recovered");
7845        assert_eq!(provider.resource_count(), 0);
7846        assert_eq!(
7847            harness.store.attempt(id).unwrap().unwrap().state(),
7848            AttemptState::Cleaned
7849        );
7850    }
7851
7852    #[tokio::test]
7853    async fn provider_resource_without_live_journal_is_surfaced_but_never_deleted() {
7854        let harness = isolated_harness();
7855        let provider = Arc::new(FakeIsolatedProvider::default());
7856        let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7857        let id = AttemptId::from_u128(0x8888);
7858        let runtime = harness.host_root().join("orphan-provider-resource");
7859        fs::create_dir_all(&runtime).unwrap();
7860        let mut attempt =
7861            RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
7862        let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7863            harness.policy.execution_policy()
7864        else {
7865            unreachable!()
7866        };
7867        attempt
7868            .allocate_execution(AttemptExecution::Isolated {
7869                provider_kind: Backend::Oci,
7870                environment_id: None,
7871                resolved_image: image.clone(),
7872                generation: "orphan-generation".into(),
7873            })
7874            .unwrap();
7875        let identity = FakeIsolatedProvider::identity(&attempt, harness.policy.host_id);
7876        provider
7877            .resources
7878            .lock()
7879            .unwrap()
7880            .insert(id, (identity, EnvironmentState::Running));
7881        launcher
7882            .recover_startup(std::slice::from_ref(&harness.policy))
7883            .await
7884            .expect("orphan cannot block host");
7885        assert_eq!(provider.resource_count(), 1);
7886        assert!(
7887            harness
7888                .events
7889                .events()
7890                .contains(&AttemptEvent::QuarantinedOwnedResource { attempt: id })
7891        );
7892        assert_eq!(
7893            provider
7894                .actions
7895                .lock()
7896                .unwrap()
7897                .iter()
7898                .filter(|action| **action == "destroy")
7899                .count(),
7900            0
7901        );
7902
7903        let EnvironmentIdentity::Isolated { environment_id, .. } = provider
7904            .resources
7905            .lock()
7906            .unwrap()
7907            .get(&id)
7908            .unwrap()
7909            .0
7910            .clone()
7911        else {
7912            unreachable!()
7913        };
7914        attempt.begin_prepare(harness.clock.now()).unwrap();
7915        attempt.prepared_environment(environment_id).unwrap();
7916        attempt.mark_prepared(harness.clock.now()).unwrap();
7917        attempt.jit_received(harness.clock.now()).unwrap();
7918        attempt.started_isolated(harness.clock.now()).unwrap();
7919        attempt
7920            .conclude(
7921                AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7922                harness.clock.now(),
7923            )
7924            .unwrap();
7925        attempt.begin_destroy(harness.clock.now()).unwrap();
7926        attempt.clean_isolated(harness.clock.now()).unwrap();
7927        harness.store.record_attempt(&attempt).unwrap();
7928        launcher
7929            .recover_startup(std::slice::from_ref(&harness.policy))
7930            .await
7931            .expect("cleaned row cannot authorize deletion");
7932        assert_eq!(provider.resource_count(), 1);
7933        assert_eq!(
7934            provider
7935                .actions
7936                .lock()
7937                .unwrap()
7938                .iter()
7939                .filter(|action| **action == "destroy")
7940                .count(),
7941            0
7942        );
7943    }
7944
7945    /// The wiring the three-hour outage needed and did not have.
7946    ///
7947    /// A root that refuses a launch does so before `record_allocation`, so no
7948    /// attempt row carries it, and the daemon's log scrubs the paths out of the
7949    /// sentence. The record this asserts is the only surface left, and
7950    /// `service status` reads it -- so if this wiring is ever dropped, the
7951    /// failure goes back to reading `runner_start_failed reason=other` once per
7952    /// poll and nothing else.
7953    #[tokio::test]
7954    async fn a_root_that_refuses_a_launch_is_recorded_and_cleared_when_one_succeeds() {
7955        use runner_manager_platform::service::{clear_runner_root_refusal, runner_root_refusals};
7956
7957        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
7958            .with_host_runner_root();
7959        harness.ready().await;
7960
7961        // A root under two directories that do not exist: refused by the
7962        // preflight, for a reason that has nothing to do with this platform, so
7963        // the assertion holds on all three.
7964        let unusable = harness
7965            ._root
7966            .path()
7967            .join("absent")
7968            .join("deeper")
7969            .join("runners");
7970        let mut host = harness.host.clone();
7971        host.runner_root_override = Some(
7972            LocalAbsolutePath::new(unusable.to_str().expect("a UTF-8 temporary path"))
7973                .expect("a local absolute host root"),
7974        );
7975        harness.store.put_host(&host).unwrap();
7976
7977        let failure = harness
7978            .launch_result()
7979            .await
7980            .expect_err("a root whose parents are missing cannot hold a runner");
7981        assert!(
7982            matches!(failure.reason, FailureReason::Other(_)),
7983            "{failure:?}"
7984        );
7985
7986        let refusals = runner_root_refusals(&harness.app_paths).expect("readable");
7987        let refusal = refusals
7988            .first()
7989            .expect("the refusal reached the one surface that can hold it");
7990        assert_eq!(refusal.policy, harness.policy.id.to_string());
7991        assert_eq!(refusal.kind, "missing_parents");
7992        assert!(
7993            refusal.root.contains("runners") && refusal.detail.contains("runners"),
7994            "the directory must be named in full: {refusal:?}"
7995        );
7996
7997        // And a root that works clears it, so a host that has been fixed stops
7998        // reporting a fault it no longer has.
7999        harness.store.put_host(&harness.host).unwrap();
8000        harness.launch().await;
8001        assert!(
8002            runner_root_refusals(&harness.app_paths)
8003                .expect("readable")
8004                .is_empty(),
8005            "a successful placement clears that policy's record"
8006        );
8007
8008        clear_runner_root_refusal(&harness.app_paths, &harness.policy.id.to_string())
8009            .expect("cleanup");
8010    }
8011
8012    #[tokio::test]
8013    async fn a_launch_reports_how_long_each_step_took() {
8014        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8015        harness.ready().await;
8016        let started = harness.launch().await;
8017        let events = harness.events.events();
8018        let launched = events
8019            .iter()
8020            .position(|event| matches!(event, AttemptEvent::Launched { attempt, .. } if *attempt == started.id))
8021            .expect("a successful launch reports its step timings");
8022        let starting = events
8023            .iter()
8024            .position(|event| matches!(event, AttemptEvent::State { attempt, state: AttemptState::Starting } if *attempt == started.id))
8025            .unwrap();
8026        assert!(starting < launched, "timings follow the start: {events:?}");
8027    }
8028
8029    #[tokio::test]
8030    async fn a_failed_launch_still_reports_its_step_timings() {
8031        let harness = Harness::new(
8032            FakeGithubLifecycle::default().fail(true),
8033            Arc::new(PersistentDemand),
8034        );
8035        harness.ready().await;
8036        let failed = harness.launch_after_waiting(Duration::from_secs(90)).await;
8037        assert!(failed.is_err());
8038
8039        let timings = harness
8040            .events
8041            .events()
8042            .into_iter()
8043            .find_map(|event| match event {
8044                AttemptEvent::Launched {
8045                    started: false,
8046                    timings,
8047                    ..
8048                } => Some(timings),
8049                _ => None,
8050            })
8051            .expect("a launch that failed at registration reports its timings");
8052        assert_eq!(timings.lock_wait, Duration::from_secs(90));
8053        assert_eq!(timings.order_wait, Duration::from_secs(2));
8054        assert_eq!(timings.spawn, Duration::ZERO, "spawn was never reached");
8055        assert!(timings.total() >= Duration::from_secs(92));
8056    }
8057
8058    #[cfg(target_os = "macos")]
8059    #[tokio::test]
8060    async fn a_launch_removes_the_seed_of_a_root_no_longer_in_use() {
8061        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8062        harness.ready().await;
8063
8064        // A runtime this host once placed under a root it no longer uses.
8065        let old_root = harness._root.path().join("old-root");
8066        let old_runtime = old_root.join("abandoned");
8067        fs::create_dir_all(old_root.join(PACKAGE_SEED_DIR).join("2.335.0")).unwrap();
8068        fs::create_dir_all(&old_runtime).unwrap();
8069        let old = RunnerAttempt::allocate(
8070            AttemptId::new_random(),
8071            harness.policy.id,
8072            &old_runtime,
8073            harness.clock.now(),
8074        );
8075        harness.store.record_attempt(&old).unwrap();
8076        let kept = harness.host_root().join(PACKAGE_SEED_DIR);
8077        fs::create_dir_all(kept.join("2.336.0")).unwrap();
8078
8079        harness.launch().await;
8080
8081        assert!(
8082            !old_root.join(PACKAGE_SEED_DIR).exists(),
8083            "the abandoned root's seed is removed"
8084        );
8085        assert!(old_runtime.exists(), "only the seed is touched");
8086        assert!(kept.exists(), "the host root's own seed is kept");
8087    }
8088
8089    #[tokio::test]
8090    async fn a_job_walks_every_state_and_cleans_every_artifact() {
8091        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8092        harness.ready().await;
8093        let started = harness.launch().await;
8094        assert_eq!(started.state(), AttemptState::Starting);
8095        assert_eq!(read_runner_id(started.runtime_path()), Some(73));
8096
8097        harness
8098            .github
8099            .observe(GithubRunnerObservation::Registered { busy: false });
8100        harness.launcher.supervise(&harness.policy).await.unwrap();
8101        assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8102
8103        harness
8104            .github
8105            .observe(GithubRunnerObservation::Registered { busy: true });
8106        harness.launcher.supervise(&harness.policy).await.unwrap();
8107        assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8108
8109        harness.processes.finish_successfully();
8110        harness
8111            .github
8112            .observe(GithubRunnerObservation::NotRegistered);
8113        harness.launcher.supervise(&harness.policy).await.unwrap();
8114        let cleaned = harness.only_attempt();
8115        assert_eq!(cleaned.state(), AttemptState::Cleaned);
8116        assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
8117        assert!(!started.runtime_path().exists());
8118        assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8119        assert_eq!(harness.github.remaining_runners.load(Ordering::SeqCst), 0);
8120
8121        let states: Vec<_> = harness
8122            .events
8123            .events()
8124            .into_iter()
8125            .filter_map(|event| match event {
8126                AttemptEvent::State { state, .. } => Some(state),
8127                _ => None,
8128            })
8129            .collect();
8130        assert_eq!(
8131            states,
8132            vec![
8133                AttemptState::Allocated,
8134                AttemptState::JitReceived,
8135                AttemptState::Starting,
8136                AttemptState::Idle,
8137                AttemptState::Busy,
8138                AttemptState::Finished,
8139                AttemptState::Cleaned,
8140            ]
8141        );
8142    }
8143
8144    #[tokio::test]
8145    async fn a_cleaned_ephemeral_attempt_reaps_a_directory_recreated_after_cleanup() {
8146        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8147        harness.ready().await;
8148        let started = harness.launch().await;
8149        let runtime = started.runtime_path().to_path_buf();
8150
8151        harness
8152            .github
8153            .observe(GithubRunnerObservation::Registered { busy: true });
8154        harness.launcher.supervise(&harness.policy).await.unwrap();
8155        harness.processes.finish_successfully();
8156        harness
8157            .github
8158            .observe(GithubRunnerObservation::NotRegistered);
8159        harness.launcher.supervise(&harness.policy).await.unwrap();
8160
8161        assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8162        assert!(!runtime.exists());
8163        assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8164
8165        // Reproduce a late orphan creating output after the successful removal
8166        // and journal transition. The next ordinary policy supervision must
8167        // remove it without replaying package release or another state change.
8168        let residue = runtime.join("_work").join("late-node-process");
8169        fs::create_dir_all(&residue).unwrap();
8170        fs::write(residue.join("node_modules.lock"), b"late residue").unwrap();
8171
8172        harness.launcher.supervise(&harness.policy).await.unwrap();
8173
8174        assert!(!runtime.exists(), "late ephemeral residue is reaped");
8175        assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8176        assert_eq!(
8177            harness.packages.releases.load(Ordering::SeqCst),
8178            1,
8179            "reaping residue does not release the package lease twice"
8180        );
8181    }
8182
8183    #[tokio::test]
8184    async fn locked_late_residue_never_blocks_startup_recovery() {
8185        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8186        harness.ready().await;
8187        let started = harness.launch().await;
8188        let runtime = started.runtime_path().to_path_buf();
8189
8190        harness
8191            .github
8192            .observe(GithubRunnerObservation::Registered { busy: true });
8193        harness.launcher.supervise(&harness.policy).await.unwrap();
8194        harness.processes.finish_successfully();
8195        harness
8196            .github
8197            .observe(GithubRunnerObservation::NotRegistered);
8198        harness.launcher.supervise(&harness.policy).await.unwrap();
8199        assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8200
8201        let held = runtime.join("_work").join("late-node-process");
8202        fs::create_dir_all(&held).unwrap();
8203        fs::write(held.join("node_modules.lock"), b"late residue").unwrap();
8204        let Some(block) = BlockedDeletion::inject(&held) else {
8205            eprintln!(
8206                "skipped: this account cannot be refused a deletion, so locked late residue cannot be injected"
8207            );
8208            return;
8209        };
8210
8211        let restarted = harness.restart();
8212        restarted
8213            .recover_startup(std::slice::from_ref(&harness.policy))
8214            .await
8215            .expect("late residue cannot take the daemon offline");
8216        assert!(runtime.exists(), "the locked residue remains for a retry");
8217        assert!(
8218            harness
8219                .reconcile_events
8220                .events()
8221                .iter()
8222                .any(|event| matches!(
8223                    event,
8224                    LifecycleEvent::AttemptCleanFailed {
8225                        attempt,
8226                        reason: "late_ephemeral_workspace_could_not_be_removed",
8227                        ..
8228                    } if *attempt == started.id
8229                )),
8230            "the non-blocking cleanup failure remains visible"
8231        );
8232
8233        block.release();
8234        // A tolerated cleanup failure is retried after its backoff delay.
8235        harness.clock.advance_secs(31);
8236        restarted
8237            .supervise(&harness.policy)
8238            .await
8239            .expect("the ordinary retry succeeds after the lock is released");
8240        assert!(!runtime.exists(), "the retry removes the late residue");
8241        assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8242        assert_eq!(
8243            harness.packages.releases.load(Ordering::SeqCst),
8244            1,
8245            "retrying residue never releases the package lease twice"
8246        );
8247    }
8248
8249    #[tokio::test]
8250    async fn mixed_locked_cleanup_residue_isolated_from_the_whole_startup_pass() {
8251        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8252        harness.ready().await;
8253
8254        // The first attempt is terminal but its initial cleanup cannot finish.
8255        // This is the state left by the original failed job on the affected
8256        // workstation.
8257        let terminal = harness.launch().await;
8258        let terminal_runtime = terminal.runtime_path().to_path_buf();
8259        harness.conclude(terminal.id);
8260        let Some(terminal_block) = BlockedDeletion::inject(&terminal_runtime.join("late-child"))
8261        else {
8262            eprintln!("skipped: this account cannot inject the two independent deletion refusals");
8263            return;
8264        };
8265
8266        // The second attempt was cleaned successfully, then a child recreated
8267        // part of `_work`. Both real-world residue shapes existed together in
8268        // the incident, and neither may short-circuit recovery of the other.
8269        let cleaned = harness.launch().await;
8270        let cleaned_runtime = cleaned.runtime_path().to_path_buf();
8271        harness.conclude(cleaned.id);
8272        harness
8273            .launcher
8274            .clean(cleaned.id)
8275            .await
8276            .expect("the second attempt initially cleans");
8277        let Some(cleaned_block) =
8278            BlockedDeletion::inject(&cleaned_runtime.join("_work").join("late-child"))
8279        else {
8280            terminal_block.release();
8281            eprintln!("skipped: this account cannot inject the two independent deletion refusals");
8282            return;
8283        };
8284
8285        let restarted = harness.restart();
8286        for _ in 0..2 {
8287            restarted
8288                .recover_startup(std::slice::from_ref(&harness.policy))
8289                .await
8290                .expect("repeated recovery remains ready while both residues are locked");
8291        }
8292        assert_ne!(
8293            harness.attempt(terminal.id).state(),
8294            AttemptState::Cleaned,
8295            "the journal does not claim the terminal workspace was removed"
8296        );
8297        assert_eq!(
8298            harness.attempt(cleaned.id).state(),
8299            AttemptState::Cleaned,
8300            "late residue does not undo an already durable cleaned transition"
8301        );
8302        assert!(terminal_runtime.exists());
8303        assert!(cleaned_runtime.exists());
8304        assert_eq!(
8305            harness.packages.releases.load(Ordering::SeqCst),
8306            1,
8307            "repeated recovery neither loses nor duplicates package leases"
8308        );
8309
8310        terminal_block.release();
8311        cleaned_block.release();
8312        // A tolerated cleanup failure is retried after its backoff delay.
8313        harness.clock.advance_secs(31);
8314        restarted
8315            .supervise(&harness.policy)
8316            .await
8317            .expect("one ordinary pass clears both residues after their locks disappear");
8318        assert!(!terminal_runtime.exists());
8319        assert!(!cleaned_runtime.exists());
8320        assert_eq!(harness.attempt(terminal.id).state(), AttemptState::Cleaned);
8321        assert_eq!(harness.attempt(cleaned.id).state(), AttemptState::Cleaned);
8322        assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 2);
8323    }
8324
8325    #[tokio::test]
8326    async fn a_terminal_attempt_whose_runtime_is_already_gone_recovers_cleanly() {
8327        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8328        harness.ready().await;
8329        let attempt = harness.launch().await;
8330        let runtime = attempt.runtime_path().to_path_buf();
8331        harness.conclude(attempt.id);
8332        fs::remove_dir_all(&runtime).unwrap();
8333
8334        harness
8335            .restart()
8336            .recover_startup(std::slice::from_ref(&harness.policy))
8337            .await
8338            .expect("an already absent disposable runtime is successful cleanup");
8339
8340        assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
8341        assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8342    }
8343
8344    #[tokio::test]
8345    async fn a_non_workspace_cleanup_failure_still_fails_closed_and_can_retry() {
8346        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8347        harness.ready().await;
8348        let attempt = harness.launch().await;
8349        let runtime = attempt.runtime_path().to_path_buf();
8350        harness.conclude(attempt.id);
8351        harness.packages.fail_releases(1);
8352
8353        let restarted = harness.restart();
8354        let failure = restarted
8355            .recover_startup(std::slice::from_ref(&harness.policy))
8356            .await
8357            .expect_err("a package-accounting failure is not safe to downgrade to residue");
8358        assert!(
8359            failure
8360                .reason()
8361                .to_string()
8362                .contains("package lease could not be released")
8363        );
8364        assert!(
8365            !runtime.exists(),
8366            "workspace removal completed before release failed"
8367        );
8368        assert_ne!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
8369
8370        restarted
8371            .recover_startup(std::slice::from_ref(&harness.policy))
8372            .await
8373            .expect("the same journal entry retries safely after the transient failure");
8374        assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
8375        assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8376    }
8377
8378    #[tokio::test]
8379    async fn startup_never_scrubs_a_cleaned_persistent_work_directory() {
8380        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
8381            .with_persistent_workspace(1);
8382        harness.ready().await;
8383        let attempt = harness.launch().await;
8384        let retained = attempt.runtime_path().join("_work").join("checkout-marker");
8385        fs::create_dir_all(retained.parent().unwrap()).unwrap();
8386        fs::write(&retained, b"persistent checkout").unwrap();
8387        harness.cleanup_retaining_work(attempt.id).await;
8388
8389        harness
8390            .restart()
8391            .recover_startup(std::slice::from_ref(&harness.policy))
8392            .await
8393            .expect("a cleaned persistent slot is already reconciled");
8394
8395        assert_eq!(fs::read(&retained).unwrap(), b"persistent checkout");
8396        assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8397    }
8398
8399    #[tokio::test]
8400    async fn an_idle_exit_is_not_a_failure_in_the_journal_or_events() {
8401        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8402        harness.ready().await;
8403        let started = harness.launch().await;
8404        harness
8405            .github
8406            .observe(GithubRunnerObservation::Registered { busy: false });
8407        harness.launcher.supervise(&harness.policy).await.unwrap();
8408        harness.clock.advance_secs(11);
8409        harness.processes.set_alive(false);
8410        harness
8411            .github
8412            .observe(GithubRunnerObservation::NotRegistered);
8413        harness.launcher.supervise(&harness.policy).await.unwrap();
8414
8415        let cleaned = harness.only_attempt();
8416        assert!(cleaned.outcome().unwrap().is_idle_exit());
8417        assert!(!cleaned.outcome().unwrap().is_failure());
8418        assert!(!started.runtime_path().exists());
8419        assert!(
8420            harness
8421                .reconcile_events
8422                .events()
8423                .iter()
8424                .any(|event| matches!(
8425                    event,
8426                    LifecycleEvent::AttemptCleaned {
8427                        outcome: OutcomeKind::IdleExit,
8428                        ..
8429                    }
8430                ))
8431        );
8432        assert!(!harness.events.events().iter().any(|event| matches!(
8433            event,
8434            AttemptEvent::Concluded {
8435                outcome: OutcomeKind::Failed,
8436                ..
8437            }
8438        )));
8439    }
8440
8441    #[tokio::test]
8442    async fn handoff_is_absent_after_success_and_every_failed_spawn_retry() {
8443        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8444        harness.processes.fail_spawns(2);
8445        harness.ready().await;
8446        let attempt = harness.launch().await;
8447        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 3);
8448        assert!(harness.processes.saw_secret.load(Ordering::SeqCst));
8449        let names: Vec<_> = fs::read_dir(attempt.runtime_path())
8450            .unwrap()
8451            .map(|entry| entry.unwrap().file_name())
8452            .collect();
8453        assert!(
8454            names.iter().all(|name| {
8455                !name
8456                    .to_string_lossy()
8457                    .starts_with(RestrictiveHandoff::NAME_PREFIX)
8458            }),
8459            "JIT artifact survived: {names:?}"
8460        );
8461        assert_eq!(
8462            *harness.delay.0.lock().unwrap(),
8463            vec![Duration::from_millis(10), Duration::from_millis(20)]
8464        );
8465    }
8466
8467    #[tokio::test]
8468    async fn jit_retry_stops_with_demand_and_a_terminal_403_never_retries() {
8469        let gone = Harness::new(
8470            FakeGithubLifecycle::default().fail(false),
8471            Arc::new(FakeDemand::answering([false])),
8472        );
8473        gone.ready().await;
8474        assert!(gone.launch_result().await.is_err());
8475        assert_eq!(gone.github.registrations.load(Ordering::SeqCst), 1);
8476        assert!(gone.delay.0.lock().unwrap().is_empty());
8477
8478        let forbidden = Harness::new(
8479            FakeGithubLifecycle::default().fail(true),
8480            Arc::new(PersistentDemand),
8481        );
8482        forbidden.ready().await;
8483        assert!(forbidden.launch_result().await.is_err());
8484        assert_eq!(forbidden.github.registrations.load(Ordering::SeqCst), 1);
8485        assert!(forbidden.delay.0.lock().unwrap().is_empty());
8486        assert!(matches!(
8487            forbidden.only_attempt().outcome(),
8488            Some(AttemptOutcome::Failed {
8489                reason: FailureReason::Other(action)
8490            }) if action.contains("403")
8491        ));
8492
8493        let transient = Harness::new(
8494            FakeGithubLifecycle::default().fail(false).fail(false),
8495            Arc::new(PersistentDemand),
8496        );
8497        transient.ready().await;
8498        transient.launch().await;
8499        assert_eq!(transient.github.registrations.load(Ordering::SeqCst), 3);
8500        assert_eq!(
8501            *transient.delay.0.lock().unwrap(),
8502            vec![Duration::from_millis(10), Duration::from_millis(20)]
8503        );
8504    }
8505
8506    /// The layout has to leave room for what the runner writes underneath it.
8507    ///
8508    /// Windows refuses a path over `MAX_PATH`, and this product's own CI hit
8509    /// that: 264 characters against a limit of 260, failing three checkout
8510    /// retries with `Filename too long`. The two identifiers in the old layout
8511    /// cost 74 characters between them for no benefit -- an attempt id is
8512    /// unique on its own.
8513    #[test]
8514    fn a_workspace_leaves_room_for_the_deepest_path_a_checkout_writes() {
8515        const MAX_PATH: usize = 260;
8516        // The real root on the machine this was found on.
8517        let root = r"C:\Users\IvanD\AppData\Local\IvanMurzak\runner-manager\data\runtime";
8518        // What `actions/checkout` writes at its deepest: the work directory,
8519        // the repository named twice, and a pack keep-file with a 40-character
8520        // object name.
8521        let repo = "GitHub-Runner-Scaler-UI";
8522        let deepest = format!(
8523            r"_work\{repo}\{repo}\.git\objects\pack\pack-{}.keep",
8524            "0".repeat(40)
8525        );
8526
8527        let name = workspace_name(AttemptId::new_random());
8528        assert_eq!(name.len(), WORKSPACE_NAME_LEN, "{name}");
8529        assert!(
8530            name.chars().all(|c| c.is_ascii_hexdigit()),
8531            "a directory name must not carry the identifier's dashes: {name}"
8532        );
8533
8534        let full = format!(r"{root}\{name}\{deepest}");
8535        assert!(
8536            full.len() < MAX_PATH,
8537            "the deepest path a checkout writes must fit: {} characters, limit {MAX_PATH}",
8538            full.len()
8539        );
8540
8541        // The discriminator: the layout this replaced does not fit, so a test
8542        // that passed for both would be proving nothing.
8543        let old = format!(
8544            r"{root}\{}\{}\{deepest}",
8545            PolicyId::new_random(),
8546            AttemptId::new_random()
8547        );
8548        assert!(
8549            old.len() > MAX_PATH,
8550            "the old layout is supposed to be the thing that did not fit: {} characters",
8551            old.len()
8552        );
8553    }
8554
8555    #[tokio::test]
8556    async fn two_attempts_never_share_a_workspace_even_after_failure() {
8557        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8558        harness.ready().await;
8559        let first = harness.launch().await;
8560        fs::write(first.runtime_path().join("hostile-leftover"), b"first job").unwrap();
8561        harness
8562            .github
8563            .observe(GithubRunnerObservation::Registered { busy: false });
8564        harness.launcher.supervise(&harness.policy).await.unwrap();
8565        harness.clock.advance_secs(11);
8566        harness.processes.set_alive(false);
8567        harness
8568            .github
8569            .observe(GithubRunnerObservation::NotRegistered);
8570        harness.launcher.supervise(&harness.policy).await.unwrap();
8571        assert!(!first.runtime_path().exists());
8572
8573        let second = harness.launch().await;
8574        assert_ne!(first.runtime_path(), second.runtime_path());
8575        assert!(!second.runtime_path().join("hostile-leftover").exists());
8576
8577        fs::write(
8578            second.runtime_path().join("hostile-on-failure"),
8579            b"second job",
8580        )
8581        .unwrap();
8582        harness.processes.set_alive(false);
8583        harness
8584            .github
8585            .observe(GithubRunnerObservation::NotRegistered);
8586        harness.launcher.supervise(&harness.policy).await.unwrap();
8587        assert!(
8588            !second.runtime_path().exists(),
8589            "failed workspace was retained"
8590        );
8591    }
8592
8593    #[tokio::test]
8594    async fn a_runner_that_never_gets_a_job_is_stopped_deregistered_and_not_replaced() {
8595        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8596        harness.ready().await;
8597        let attempt = harness.launch().await;
8598
8599        // Registered and waiting, which is where it stays: the fake keeps
8600        // answering the same observation, exactly as GitHub does for a runner
8601        // nobody assigns work to.
8602        harness
8603            .github
8604            .observe(GithubRunnerObservation::Registered { busy: false });
8605        harness.launcher.supervise(&harness.policy).await.unwrap();
8606        assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8607
8608        // One second inside the ten-second idle timeout nothing happens, which
8609        // is what keeps this from being a test that would pass on any clock.
8610        harness.clock.advance_secs(9);
8611        harness
8612            .github
8613            .observe(GithubRunnerObservation::Registered { busy: false });
8614        let none_yet = harness.launcher.supervise(&harness.policy).await.unwrap();
8615        assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8616        assert!(none_yet.is_empty());
8617        assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 0);
8618
8619        // Past it, the agent ends the runner itself.
8620        harness.clock.advance_secs(1);
8621        harness
8622            .github
8623            .observe(GithubRunnerObservation::Registered { busy: false });
8624        let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8625
8626        let concluded = harness.store.attempt(attempt.id).unwrap().unwrap();
8627        assert_eq!(
8628            concluded.outcome(),
8629            Some(&AttemptOutcome::ExitedIdleWithoutWork),
8630            "a surplus runner did not fail; recording one as a failure sends an operator \
8631             hunting a fault that does not exist"
8632        );
8633        assert_eq!(concluded.state(), AttemptState::Cleaned);
8634        assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 1);
8635        assert!(!attempt.runtime_path().exists());
8636
8637        // The registration goes with it. Without this the runner stays listed
8638        // in the target's runner settings after the process it named is gone.
8639        assert_eq!(
8640            *harness.github.deregistrations.lock().unwrap(),
8641            vec![73],
8642            "the attempt's own runner id, deleted exactly once"
8643        );
8644
8645        // And nothing is started in its place: the work it was launched for went
8646        // elsewhere, so a replacement would rebuild it every idle timeout.
8647        assert!(
8648            replacements.is_empty(),
8649            "a surplus exit must not request a replacement"
8650        );
8651    }
8652
8653    #[tokio::test]
8654    async fn a_registration_github_will_not_delete_still_concludes_the_attempt() {
8655        // The delete is best-effort by construction: the process is gone and the
8656        // slot has to come back. Holding the conclusion until GitHub cooperates
8657        // would leak a capacity slot on every unreachable moment.
8658        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8659        harness.ready().await;
8660        let attempt = harness.launch().await;
8661        harness
8662            .github
8663            .observe(GithubRunnerObservation::Registered { busy: false });
8664        harness.launcher.supervise(&harness.policy).await.unwrap();
8665
8666        harness
8667            .github
8668            .deregistration_fails
8669            .store(true, Ordering::SeqCst);
8670        harness.clock.advance_secs(11);
8671        harness
8672            .github
8673            .observe(GithubRunnerObservation::Registered { busy: false });
8674        harness.launcher.supervise(&harness.policy).await.unwrap();
8675
8676        assert_eq!(
8677            *harness.github.deregistrations.lock().unwrap(),
8678            vec![73],
8679            "the delete was attempted"
8680        );
8681        let concluded = harness.store.attempt(attempt.id).unwrap().unwrap();
8682        assert_eq!(
8683            concluded.outcome(),
8684            Some(&AttemptOutcome::ExitedIdleWithoutWork),
8685            "the attempt concluded anyway"
8686        );
8687        assert_eq!(concluded.state(), AttemptState::Cleaned);
8688        assert!(!attempt.runtime_path().exists());
8689    }
8690
8691    #[tokio::test]
8692    async fn exit_before_acceptance_returns_replacement_intent_without_launching() {
8693        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8694        harness.ready().await;
8695        let first = harness.launch().await;
8696        harness.processes.set_alive(false);
8697        harness
8698            .github
8699            .observe(GithubRunnerObservation::NotRegistered);
8700        let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8701        let failed = harness.store.attempt(first.id).unwrap().unwrap();
8702        assert!(matches!(
8703            failed.outcome(),
8704            Some(AttemptOutcome::Failed {
8705                reason: FailureReason::ProcessExitedUnexpectedly
8706            })
8707        ));
8708        assert!(!first.runtime_path().exists());
8709
8710        assert_eq!(
8711            replacements,
8712            vec![ReplacementIntent {
8713                policy: harness.policy.id,
8714                previous_attempt: first.id,
8715                operation: "exit_before_acceptance_replacement",
8716            }]
8717        );
8718        assert_eq!(harness.store.attempts().unwrap().len(), 1);
8719        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 1);
8720        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 1);
8721        assert!(harness.delay.0.lock().unwrap().is_empty());
8722    }
8723
8724    /// Writes the runner's own diagnostics: always the listener's log, and the
8725    /// job worker's when `ran_a_job`. `Runner.Listener` starts `Runner.Worker`
8726    /// only for a job it has acquired.
8727    fn write_runner_diagnostics(runtime: &Path, ran_a_job: bool) {
8728        let diag = runtime.join("_diag");
8729        fs::create_dir_all(&diag).unwrap();
8730        fs::write(diag.join("Runner_20261007-211941-utc.log"), b"listener").unwrap();
8731        if ran_a_job {
8732            fs::write(diag.join("Worker_20261007-211950-utc.log"), b"worker").unwrap();
8733        }
8734    }
8735
8736    /// The macOS field case: a job of a few seconds starts and ends between two
8737    /// passes, so GitHub was never seen reporting the runner busy -- the attempt
8738    /// is still `starting` -- and the registration is already gone. It used to be
8739    /// recorded `process_exited_unexpectedly`, with a replacement runner.
8740    #[tokio::test]
8741    async fn a_job_too_short_to_be_seen_busy_is_recorded_as_completed() {
8742        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8743        harness.ready().await;
8744        let started = harness.launch().await;
8745        assert_eq!(started.state(), AttemptState::Starting);
8746        write_runner_diagnostics(started.runtime_path(), true);
8747        harness.processes.finish_successfully();
8748        harness
8749            .github
8750            .observe(GithubRunnerObservation::NotRegistered);
8751
8752        let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8753
8754        let cleaned = harness.only_attempt();
8755        assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
8756        assert_eq!(cleaned.state(), AttemptState::Cleaned);
8757        assert_eq!(cleaned.github_runner_id(), Some(73));
8758        assert!(
8759            replacements.is_empty(),
8760            "a finished job needs no replacement"
8761        );
8762        let busy_seen = harness.events.events().into_iter().any(|event| {
8763            matches!(
8764                event,
8765                AttemptEvent::State {
8766                    state: AttemptState::Busy,
8767                    ..
8768                }
8769            )
8770        });
8771        assert!(busy_seen, "the journal walks the diagram to `busy` first");
8772    }
8773
8774    /// The same from `idle`: seen registered and waiting once, then gone.
8775    #[tokio::test]
8776    async fn a_job_taken_between_two_passes_after_idle_is_recorded_as_completed() {
8777        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8778        harness.ready().await;
8779        let started = harness.launch().await;
8780        harness
8781            .github
8782            .observe(GithubRunnerObservation::Registered { busy: false });
8783        harness.launcher.supervise(&harness.policy).await.unwrap();
8784        assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8785
8786        write_runner_diagnostics(started.runtime_path(), true);
8787        harness.processes.finish_successfully();
8788        harness
8789            .github
8790            .observe(GithubRunnerObservation::NotRegistered);
8791        let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8792
8793        assert_eq!(
8794            harness.only_attempt().outcome(),
8795            Some(&AttemptOutcome::CompletedJob)
8796        );
8797        assert!(replacements.is_empty());
8798    }
8799
8800    /// The reverse error, from the same sampling: GitHub showed the runner
8801    /// assigned, the runner never started the job (no worker log), and it exited
8802    /// 0. That is not a completed job.
8803    #[tokio::test]
8804    async fn an_assignment_the_runner_never_took_up_is_not_a_completed_job() {
8805        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8806        harness.ready().await;
8807        let started = harness.launch().await;
8808        harness
8809            .github
8810            .observe(GithubRunnerObservation::Registered { busy: true });
8811        harness.launcher.supervise(&harness.policy).await.unwrap();
8812        assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8813
8814        write_runner_diagnostics(started.runtime_path(), false);
8815        harness.processes.finish_successfully();
8816        harness
8817            .github
8818            .observe(GithubRunnerObservation::NotRegistered);
8819        harness.launcher.supervise(&harness.policy).await.unwrap();
8820
8821        let cleaned = harness.only_attempt();
8822        assert_eq!(
8823            cleaned.outcome(),
8824            Some(&AttemptOutcome::ExitedIdleWithoutWork)
8825        );
8826        assert_eq!(cleaned.state(), AttemptState::Cleaned);
8827    }
8828
8829    /// The mirror case: the runner really ran its job and exited 0 before GitHub
8830    /// took it off the list. It used to be recorded `orphaned`.
8831    #[tokio::test]
8832    async fn a_finished_job_still_listed_at_github_is_completed_and_deregistered() {
8833        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8834        harness.ready().await;
8835        let started = harness.launch().await;
8836        harness
8837            .github
8838            .observe(GithubRunnerObservation::Registered { busy: true });
8839        harness.launcher.supervise(&harness.policy).await.unwrap();
8840
8841        write_runner_diagnostics(started.runtime_path(), true);
8842        harness.processes.finish_successfully();
8843        harness
8844            .github
8845            .observe(GithubRunnerObservation::Registered { busy: true });
8846        harness.launcher.supervise(&harness.policy).await.unwrap();
8847
8848        assert_eq!(
8849            harness.only_attempt().outcome(),
8850            Some(&AttemptOutcome::CompletedJob)
8851        );
8852        assert_eq!(*harness.github.deregistrations.lock().unwrap(), vec![73]);
8853    }
8854
8855    /// A live runner journalled `busy` whose assignment GitHub withdrew goes
8856    /// back to `idle`, where the idle timeout holds it again.
8857    #[tokio::test]
8858    async fn a_live_runner_whose_assignment_was_withdrawn_goes_back_to_idle() {
8859        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8860        harness.ready().await;
8861        let started = harness.launch().await;
8862        harness
8863            .github
8864            .observe(GithubRunnerObservation::Registered { busy: true });
8865        harness.launcher.supervise(&harness.policy).await.unwrap();
8866        assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8867
8868        write_runner_diagnostics(started.runtime_path(), false);
8869        harness
8870            .github
8871            .observe(GithubRunnerObservation::Registered { busy: false });
8872        harness.launcher.supervise(&harness.policy).await.unwrap();
8873        assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8874
8875        // A runner that did start its job keeps `busy` whatever GitHub says.
8876        harness
8877            .github
8878            .observe(GithubRunnerObservation::Registered { busy: true });
8879        harness.launcher.supervise(&harness.policy).await.unwrap();
8880        write_runner_diagnostics(started.runtime_path(), true);
8881        harness
8882            .github
8883            .observe(GithubRunnerObservation::Registered { busy: false });
8884        harness.launcher.supervise(&harness.policy).await.unwrap();
8885        assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8886    }
8887
8888    /// While GitHub cannot be reached nothing is concluded, so a registration
8889    /// still listed there is removed once it answers.
8890    #[tokio::test]
8891    async fn a_clean_exit_waits_for_github_to_answer() {
8892        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8893        harness.ready().await;
8894        let started = harness.launch().await;
8895        write_runner_diagnostics(started.runtime_path(), true);
8896        harness.processes.finish_successfully();
8897        harness.github.observe(GithubRunnerObservation::Unreachable);
8898        harness.launcher.supervise(&harness.policy).await.unwrap();
8899        assert_eq!(harness.only_attempt().state(), AttemptState::Starting);
8900
8901        harness
8902            .github
8903            .observe(GithubRunnerObservation::NotRegistered);
8904        harness.launcher.supervise(&harness.policy).await.unwrap();
8905        assert_eq!(
8906            harness.only_attempt().outcome(),
8907            Some(&AttemptOutcome::CompletedJob)
8908        );
8909    }
8910
8911    /// A runner id nobody can name is not a reason to stop supervising the
8912    /// policy: the attempt is concluded the way it was before, without one.
8913    #[tokio::test]
8914    async fn a_short_job_with_no_runner_id_still_concludes() {
8915        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8916        harness.ready().await;
8917        let started = harness.launch().await;
8918        fs::remove_file(started.runtime_path().join(RUNNER_ID_FILE)).unwrap();
8919        write_runner_diagnostics(started.runtime_path(), true);
8920        harness.processes.finish_successfully();
8921        harness
8922            .github
8923            .observe(GithubRunnerObservation::NotRegistered);
8924        harness
8925            .launcher
8926            .supervise(&harness.policy)
8927            .await
8928            .expect("supervision goes on");
8929        assert!(harness.only_attempt().is_terminal());
8930    }
8931
8932    /// Without the runner's diagnostics nothing changes: a clean exit seen only
8933    /// at `starting` is still a runner that exited before taking a job.
8934    #[tokio::test]
8935    async fn without_runner_diagnostics_an_early_exit_is_still_a_failure() {
8936        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8937        harness.ready().await;
8938        harness.launch().await;
8939        harness.processes.finish_successfully();
8940        harness
8941            .github
8942            .observe(GithubRunnerObservation::NotRegistered);
8943        harness.launcher.supervise(&harness.policy).await.unwrap();
8944        assert!(matches!(
8945            harness.only_attempt().outcome(),
8946            Some(AttemptOutcome::Failed {
8947                reason: FailureReason::ProcessExitedUnexpectedly
8948            })
8949        ));
8950    }
8951
8952    #[tokio::test]
8953    async fn expired_jit_is_removed_and_does_not_reregister_after_demand_disappears() {
8954        let harness = Harness::new(
8955            FakeGithubLifecycle::default(),
8956            Arc::new(FakeDemand::answering([false])),
8957        );
8958        let id = AttemptId::new_random();
8959        let runtime = harness
8960            .launcher
8961            .app_paths
8962            .runtime_dir()
8963            .join(harness.policy.id.to_string())
8964            .join(id.to_string());
8965        fs::create_dir_all(&runtime).unwrap();
8966        let mut attempt =
8967            RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
8968        attempt.jit_received(harness.clock.now()).unwrap();
8969        harness.store.record_attempt(&attempt).unwrap();
8970        harness.clock.advance_secs(11);
8971        let replacements = harness
8972            .launcher
8973            .recover_startup(std::slice::from_ref(&harness.policy))
8974            .await
8975            .unwrap();
8976        assert_eq!(
8977            replacements,
8978            vec![ReplacementIntent {
8979                policy: harness.policy.id,
8980                previous_attempt: id,
8981                operation: "jit_expired_replacement",
8982            }]
8983        );
8984
8985        let cleaned = harness.store.attempt(id).unwrap().unwrap();
8986        assert_eq!(cleaned.state(), AttemptState::Cleaned);
8987        assert!(matches!(
8988            cleaned.outcome(),
8989            Some(AttemptOutcome::Failed {
8990                reason: FailureReason::JitExpired
8991            })
8992        ));
8993        assert!(!runtime.exists());
8994        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
8995        assert!(harness.delay.0.lock().unwrap().is_empty());
8996    }
8997
8998    #[tokio::test]
8999    async fn expired_jit_returns_intent_but_never_launches_inside_lifecycle() {
9000        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9001        let id = AttemptId::new_random();
9002        let runtime = harness
9003            .launcher
9004            .app_paths
9005            .runtime_dir()
9006            .join("expired-with-demand");
9007        fs::create_dir_all(&runtime).unwrap();
9008        let mut attempt =
9009            RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9010        attempt.jit_received(harness.clock.now()).unwrap();
9011        harness.store.record_attempt(&attempt).unwrap();
9012        harness.clock.advance_secs(11);
9013        let replacements = harness
9014            .launcher
9015            .recover_startup(std::slice::from_ref(&harness.policy))
9016            .await
9017            .unwrap();
9018
9019        let attempts = harness.store.attempts().unwrap();
9020        assert_eq!(attempts.len(), 1);
9021        assert_eq!(
9022            attempts
9023                .iter()
9024                .find(|attempt| attempt.id == id)
9025                .unwrap()
9026                .state(),
9027            AttemptState::Cleaned
9028        );
9029        assert_eq!(
9030            replacements,
9031            vec![ReplacementIntent {
9032                policy: harness.policy.id,
9033                previous_attempt: id,
9034                operation: "jit_expired_replacement",
9035            }]
9036        );
9037        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
9038        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
9039        assert!(harness.delay.0.lock().unwrap().is_empty());
9040    }
9041
9042    #[tokio::test]
9043    async fn package_materialization_retries_are_bounded_and_demand_adjacent() {
9044        let persistent = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9045        persistent.packages.fail_materializations(2);
9046        persistent.ready().await;
9047        persistent.launch().await;
9048        assert_eq!(
9049            persistent.packages.materializations.load(Ordering::SeqCst),
9050            3
9051        );
9052        assert_eq!(
9053            *persistent.delay.0.lock().unwrap(),
9054            vec![Duration::from_millis(10), Duration::from_millis(20)]
9055        );
9056
9057        let gone_before_wait = Harness::new(
9058            FakeGithubLifecycle::default(),
9059            Arc::new(FakeDemand::answering([false])),
9060        );
9061        gone_before_wait.packages.fail_materializations(3);
9062        gone_before_wait.ready().await;
9063        assert!(gone_before_wait.launch_result().await.is_err());
9064        assert_eq!(
9065            gone_before_wait
9066                .packages
9067                .materializations
9068                .load(Ordering::SeqCst),
9069            1
9070        );
9071        assert!(gone_before_wait.delay.0.lock().unwrap().is_empty());
9072
9073        let gone_during_wait = Harness::new(
9074            FakeGithubLifecycle::default(),
9075            Arc::new(FakeDemand::answering([true, false])),
9076        );
9077        gone_during_wait.packages.fail_materializations(3);
9078        gone_during_wait.ready().await;
9079        assert!(gone_during_wait.launch_result().await.is_err());
9080        assert_eq!(
9081            gone_during_wait
9082                .packages
9083                .materializations
9084                .load(Ordering::SeqCst),
9085            1
9086        );
9087        assert_eq!(
9088            *gone_during_wait.delay.0.lock().unwrap(),
9089            vec![Duration::from_millis(10)]
9090        );
9091    }
9092
9093    #[tokio::test]
9094    async fn replacement_is_intent_only_and_never_launches_inside_lifecycle() {
9095        let harness = Harness::new(
9096            FakeGithubLifecycle::default(),
9097            Arc::new(FakeDemand::answering([true, false])),
9098        );
9099        harness.ready().await;
9100        let first = harness.launch().await;
9101        harness.processes.set_alive(false);
9102        harness
9103            .github
9104            .observe(GithubRunnerObservation::NotRegistered);
9105        let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
9106
9107        assert_eq!(harness.store.attempts().unwrap().len(), 1);
9108        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 1);
9109        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 1);
9110        assert!(harness.delay.0.lock().unwrap().is_empty());
9111        assert_eq!(
9112            replacements,
9113            vec![ReplacementIntent {
9114                policy: harness.policy.id,
9115                previous_attempt: first.id,
9116                operation: "exit_before_acceptance_replacement",
9117            }]
9118        );
9119        assert_eq!(
9120            harness.store.attempt(first.id).unwrap().unwrap().state(),
9121            AttemptState::Cleaned
9122        );
9123    }
9124
9125    #[tokio::test]
9126    async fn startup_adopts_a_live_process_and_refuses_launch_before_recovery() {
9127        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9128        let before = harness.launch_result().await;
9129        assert!(before.is_err());
9130        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
9131
9132        let id = AttemptId::new_random();
9133        let runtime = harness.launcher.app_paths.runtime_dir().join("adopt");
9134        fs::create_dir_all(&runtime).unwrap();
9135        let mut attempt =
9136            RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
9137        attempt.jit_received(harness.clock.now()).unwrap();
9138        attempt.started(4242, harness.clock.now()).unwrap();
9139        harness.store.record_attempt(&attempt).unwrap();
9140        harness.processes.set_alive(true);
9141        harness
9142            .github
9143            .observe(GithubRunnerObservation::NotRegistered);
9144        let replacements = harness
9145            .launcher
9146            .recover_startup(std::slice::from_ref(&harness.policy))
9147            .await
9148            .unwrap();
9149        assert!(replacements.is_empty());
9150        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
9151        assert!(
9152            harness
9153                .events
9154                .events()
9155                .contains(&AttemptEvent::Adopted { attempt: id })
9156        );
9157    }
9158
9159    #[tokio::test]
9160    async fn spawn_before_starting_crash_recovers_pid_then_completes_and_cleans() {
9161        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9162        let id = AttemptId::new_random();
9163        let runtime = harness
9164            .launcher
9165            .app_paths
9166            .runtime_dir()
9167            .join("spawn-before-starting");
9168        fs::create_dir_all(&runtime).unwrap();
9169        let mut attempt =
9170            RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9171        attempt.jit_received(harness.clock.now()).unwrap();
9172        harness.store.record_attempt(&attempt).unwrap();
9173        harness.processes.set_alive(true);
9174        harness
9175            .github
9176            .observe(GithubRunnerObservation::Registered { busy: true });
9177
9178        let replacements = harness
9179            .launcher
9180            .recover_startup(std::slice::from_ref(&harness.policy))
9181            .await
9182            .unwrap();
9183        assert!(replacements.is_empty());
9184        let recovered = harness.store.attempt(id).unwrap().unwrap();
9185        assert_eq!(recovered.state(), AttemptState::Busy);
9186        assert_eq!(recovered.process_id(), Some(4242));
9187        assert_eq!(recovered.github_runner_id(), Some(73));
9188        let events = harness.events.events();
9189        let starting = events
9190            .iter()
9191            .position(|event| matches!(event, AttemptEvent::State { attempt, state: AttemptState::Starting } if *attempt == id))
9192            .unwrap();
9193        let busy = events
9194            .iter()
9195            .position(|event| matches!(event, AttemptEvent::State { attempt, state: AttemptState::Busy } if *attempt == id))
9196            .unwrap();
9197        assert!(starting < busy, "recovery skipped a legal edge: {events:?}");
9198
9199        harness.processes.finish_successfully();
9200        harness
9201            .github
9202            .observe(GithubRunnerObservation::NotRegistered);
9203        assert!(
9204            harness
9205                .launcher
9206                .supervise(&harness.policy)
9207                .await
9208                .unwrap()
9209                .is_empty()
9210        );
9211        let cleaned = harness.store.attempt(id).unwrap().unwrap();
9212        assert_eq!(cleaned.state(), AttemptState::Cleaned);
9213        assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
9214        assert!(!runtime.exists());
9215    }
9216
9217    #[tokio::test]
9218    async fn failed_post_spawn_stop_keeps_capacity_until_supervision_proves_death() {
9219        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9220        harness.processes.fail_spawn_with_live_child();
9221        harness.ready().await;
9222        assert!(harness.launch_result().await.is_err());
9223
9224        let attempt = harness.only_attempt();
9225        assert_eq!(attempt.state(), AttemptState::Starting);
9226        assert_eq!(attempt.process_id(), Some(4242));
9227        assert!(attempt.outcome().is_none());
9228        assert!(attempt.state().counts_against_capacity());
9229        assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 1);
9230        assert!(harness.delay.0.lock().unwrap().is_empty());
9231
9232        harness.processes.set_alive(false);
9233        harness
9234            .github
9235            .observe(GithubRunnerObservation::NotRegistered);
9236        let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
9237        assert_eq!(replacements.len(), 1);
9238        assert_eq!(
9239            harness.store.attempt(attempt.id).unwrap().unwrap().state(),
9240            AttemptState::Cleaned
9241        );
9242    }
9243
9244    #[tokio::test]
9245    async fn remote_runner_identity_closes_both_sides_of_the_registration_crash_boundary() {
9246        for sidecar_already_present in [false, true] {
9247            let harness = Harness::new(
9248                FakeGithubLifecycle::default(),
9249                Arc::new(FakeDemand::answering([false])),
9250            );
9251            let id = AttemptId::new_random();
9252            let runtime =
9253                harness
9254                    .launcher
9255                    .app_paths
9256                    .runtime_dir()
9257                    .join(if sidecar_already_present {
9258                        "after-id-sidecar"
9259                    } else {
9260                        "before-id-sidecar"
9261                    });
9262            fs::create_dir_all(&runtime).unwrap();
9263            let mut attempt =
9264                RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9265            if sidecar_already_present {
9266                write_runner_id(&runtime, 73).unwrap();
9267                attempt.jit_received(harness.clock.now()).unwrap();
9268            }
9269            harness.store.record_attempt(&attempt).unwrap();
9270            harness.processes.set_alive(true);
9271            harness
9272                .github
9273                .observe(GithubRunnerObservation::Registered { busy: false });
9274            harness
9275                .launcher
9276                .recover_startup(std::slice::from_ref(&harness.policy))
9277                .await
9278                .unwrap();
9279
9280            assert_eq!(read_runner_id(&runtime), Some(73));
9281            assert!(
9282                harness
9283                    .store
9284                    .attempt(id)
9285                    .unwrap()
9286                    .unwrap()
9287                    .outcome()
9288                    .is_none()
9289            );
9290            let events = harness.events.events();
9291            let recovered = events.iter().position(|event| {
9292                matches!(
9293                    event,
9294                    AttemptEvent::RemoteIdentityRecovered {
9295                        attempt,
9296                        runner_id: 73
9297                    } if *attempt == id
9298                )
9299            });
9300            assert_eq!(recovered.is_some(), !sidecar_already_present);
9301            if let Some(recovered) = recovered {
9302                let adopted = events
9303                    .iter()
9304                    .position(|event| matches!(event, AttemptEvent::Adopted { attempt } if *attempt == id))
9305                    .unwrap();
9306                assert!(
9307                    recovered < adopted,
9308                    "identity was not durable before adoption: {events:?}"
9309                );
9310            }
9311            assert!(runtime.exists());
9312        }
9313    }
9314
9315    #[tokio::test]
9316    async fn recovery_stays_closed_for_unknown_policy_and_unreachable_attempts() {
9317        let unknown = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9318        let unknown_attempt = RunnerAttempt::allocate(
9319            AttemptId::new_random(),
9320            PolicyId::from_u128(0xfeed),
9321            unknown
9322                .launcher
9323                .app_paths
9324                .runtime_dir()
9325                .join("unknown-policy"),
9326            unknown.clock.now(),
9327        );
9328        unknown.store.record_attempt(&unknown_attempt).unwrap();
9329        let expired_id = AttemptId::new_random();
9330        let expired_runtime = unknown
9331            .launcher
9332            .app_paths
9333            .runtime_dir()
9334            .join("expired-beside-unknown");
9335        fs::create_dir_all(&expired_runtime).unwrap();
9336        let mut expired = RunnerAttempt::allocate(
9337            expired_id,
9338            unknown.policy.id,
9339            expired_runtime,
9340            unknown.clock.now(),
9341        );
9342        expired.jit_received(unknown.clock.now()).unwrap();
9343        unknown.store.record_attempt(&expired).unwrap();
9344        unknown.clock.advance_secs(11);
9345        assert!(matches!(
9346            unknown
9347                .launcher
9348                .recover_startup(std::slice::from_ref(&unknown.policy))
9349                .await,
9350            Err(LifecycleError::RecoveryIncomplete)
9351        ));
9352        assert!(unknown.launch_result().await.is_err());
9353        assert_eq!(unknown.processes.spawns.load(Ordering::SeqCst), 0);
9354        let recovered_policy = fixtures::policy()
9355            .id(PolicyId::from_u128(0xfeed))
9356            .repository("octo/repo")
9357            .autoscale("home", 2)
9358            .active()
9359            .build();
9360        let pending = unknown
9361            .launcher
9362            .recover_startup(&[unknown.policy.clone(), recovered_policy])
9363            .await
9364            .unwrap();
9365        assert_eq!(
9366            pending,
9367            vec![ReplacementIntent {
9368                policy: unknown.policy.id,
9369                previous_attempt: expired_id,
9370                operation: "jit_expired_replacement",
9371            }]
9372        );
9373        assert_eq!(unknown.processes.spawns.load(Ordering::SeqCst), 0);
9374
9375        let unreachable = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9376        let id = AttemptId::new_random();
9377        let runtime = unreachable
9378            .launcher
9379            .app_paths
9380            .runtime_dir()
9381            .join("unreachable");
9382        fs::create_dir_all(&runtime).unwrap();
9383        unreachable
9384            .store
9385            .record_attempt(&RunnerAttempt::allocate(
9386                id,
9387                unreachable.policy.id,
9388                runtime,
9389                unreachable.clock.now(),
9390            ))
9391            .unwrap();
9392        unreachable
9393            .github
9394            .observe(GithubRunnerObservation::Unreachable);
9395        assert!(matches!(
9396            unreachable
9397                .launcher
9398                .recover_startup(std::slice::from_ref(&unreachable.policy))
9399                .await,
9400            Err(LifecycleError::RecoveryIncomplete)
9401        ));
9402        assert!(unreachable.launch_result().await.is_err());
9403        assert_eq!(unreachable.processes.spawns.load(Ordering::SeqCst), 0);
9404    }
9405
9406    #[tokio::test]
9407    async fn a_dead_busy_process_unknown_to_github_is_orphaned_and_cleaned() {
9408        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9409        let id = AttemptId::new_random();
9410        let runtime = harness.launcher.app_paths.runtime_dir().join("orphan");
9411        fs::create_dir_all(&runtime).unwrap();
9412        let mut attempt =
9413            RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9414        attempt.jit_received(harness.clock.now()).unwrap();
9415        attempt.started(4242, harness.clock.now()).unwrap();
9416        attempt.assigned_job(73, harness.clock.now()).unwrap();
9417        harness.store.record_attempt(&attempt).unwrap();
9418        harness.processes.set_alive(false);
9419        harness
9420            .github
9421            .observe(GithubRunnerObservation::NotRegistered);
9422        harness
9423            .launcher
9424            .recover_startup(std::slice::from_ref(&harness.policy))
9425            .await
9426            .unwrap();
9427        let cleaned = harness.store.attempt(id).unwrap().unwrap();
9428        assert_eq!(cleaned.state(), AttemptState::Cleaned);
9429        assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::Orphaned));
9430        assert!(!runtime.exists());
9431    }
9432
9433    #[tokio::test]
9434    async fn registration_timeout_journals_intent_stops_then_concludes_with_dead_reason() {
9435        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9436        harness.ready().await;
9437        let id = AttemptId::new_random();
9438        let runtime = harness.launcher.app_paths.runtime_dir().join("timeout");
9439        fs::create_dir_all(&runtime).unwrap();
9440        let mut attempt =
9441            RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
9442        attempt.jit_received(harness.clock.now()).unwrap();
9443        attempt.started(4242, harness.clock.now()).unwrap();
9444        harness.store.record_attempt(&attempt).unwrap();
9445        harness.clock.advance_secs(11);
9446        harness.processes.set_alive(true);
9447        harness
9448            .github
9449            .observe(GithubRunnerObservation::NotRegistered);
9450        let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
9451        assert_eq!(
9452            replacements,
9453            vec![ReplacementIntent {
9454                policy: harness.policy.id,
9455                previous_attempt: id,
9456                operation: "registration_timeout_replacement",
9457            }]
9458        );
9459
9460        assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 1);
9461        assert!(!harness.processes.alive.load(Ordering::SeqCst));
9462        let actions = harness.processes.actions.lock().unwrap().clone();
9463        let intent = actions
9464            .iter()
9465            .position(|action| *action == "terminate_intent")
9466            .unwrap();
9467        let signal = actions
9468            .iter()
9469            .position(|action| *action == "terminate")
9470            .unwrap();
9471        assert!(
9472            intent < signal,
9473            "intent was not durable before signal: {actions:?}"
9474        );
9475
9476        let cleaned = harness.store.attempt(id).unwrap().unwrap();
9477        assert!(matches!(
9478            cleaned.outcome(),
9479            Some(AttemptOutcome::Failed {
9480                reason: FailureReason::TerminatedAfterRegistrationTimeout
9481            })
9482        ));
9483        let events = harness.events.events();
9484        let intent = events
9485            .iter()
9486            .position(|event| matches!(event, AttemptEvent::TerminateIntent { .. }))
9487            .unwrap();
9488        let stopped = events
9489            .iter()
9490            .position(|event| matches!(event, AttemptEvent::Terminated { .. }))
9491            .unwrap();
9492        let concluded = events
9493            .iter()
9494            .position(|event| matches!(event, AttemptEvent::Concluded { .. }))
9495            .unwrap();
9496        assert!(intent < stopped && stopped < concluded, "{events:?}");
9497    }
9498
9499    #[tokio::test]
9500    async fn timeout_crash_recovery_returns_the_same_replacement_intent() {
9501        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9502        let id = AttemptId::new_random();
9503        let runtime = harness
9504            .launcher
9505            .app_paths
9506            .runtime_dir()
9507            .join("timeout-after-crash");
9508        fs::create_dir_all(&runtime).unwrap();
9509        let mut attempt =
9510            RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
9511        attempt.jit_received(harness.clock.now()).unwrap();
9512        attempt.started(4242, harness.clock.now()).unwrap();
9513        harness.store.record_attempt(&attempt).unwrap();
9514        harness.processes.intent.store(true, Ordering::SeqCst);
9515        harness.processes.set_alive(false);
9516        harness
9517            .github
9518            .observe(GithubRunnerObservation::NotRegistered);
9519
9520        let replacements = harness
9521            .launcher
9522            .recover_startup(std::slice::from_ref(&harness.policy))
9523            .await
9524            .unwrap();
9525        assert_eq!(
9526            replacements,
9527            vec![ReplacementIntent {
9528                policy: harness.policy.id,
9529                previous_attempt: id,
9530                operation: "registration_timeout_replacement",
9531            }]
9532        );
9533        let consumed = RunnerLauncher::supervise(&harness.launcher, &harness.policy)
9534            .await
9535            .unwrap();
9536        assert_eq!(consumed, replacements);
9537        assert!(
9538            RunnerLauncher::supervise(&harness.launcher, &harness.policy)
9539                .await
9540                .unwrap()
9541                .is_empty(),
9542            "startup replacement evidence must be consumed exactly once by e1"
9543        );
9544        assert!(matches!(
9545            harness.store.attempt(id).unwrap().unwrap().outcome(),
9546            Some(AttemptOutcome::Failed {
9547                reason: FailureReason::TerminatedAfterRegistrationTimeout
9548            })
9549        ));
9550    }
9551
9552    #[tokio::test]
9553    async fn terminate_intent_sync_failure_prevents_signal_and_conclusion() {
9554        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9555        let id = AttemptId::new_random();
9556        let runtime = harness
9557            .launcher
9558            .app_paths
9559            .runtime_dir()
9560            .join("timeout-sync-failure");
9561        fs::create_dir_all(&runtime).unwrap();
9562        let mut attempt =
9563            RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9564        attempt.jit_received(harness.clock.now()).unwrap();
9565        attempt.started(4242, harness.clock.now()).unwrap();
9566        harness.store.record_attempt(&attempt).unwrap();
9567        harness.clock.advance_secs(11);
9568        harness.processes.set_alive(true);
9569        harness.processes.fail_intent();
9570        harness
9571            .github
9572            .observe(GithubRunnerObservation::NotRegistered);
9573
9574        assert!(
9575            harness
9576                .launcher
9577                .recover_startup(std::slice::from_ref(&harness.policy))
9578                .await
9579                .is_err()
9580        );
9581        assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 0);
9582        assert!(harness.processes.alive.load(Ordering::SeqCst));
9583        assert_eq!(
9584            harness.store.attempt(id).unwrap().unwrap().state(),
9585            AttemptState::Starting
9586        );
9587        assert!(!harness.events.events().iter().any(|event| matches!(
9588            event,
9589            AttemptEvent::Terminated { attempt } | AttemptEvent::Concluded { attempt, .. }
9590                if *attempt == id
9591        )));
9592    }
9593
9594    #[tokio::test]
9595    async fn diagnostics_survive_cleanup_without_the_jit_or_a_token() {
9596        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9597        harness.ready().await;
9598        let attempt = harness.launch().await;
9599        harness
9600            .github
9601            .observe(GithubRunnerObservation::Registered { busy: false });
9602        harness.launcher.supervise(&harness.policy).await.unwrap();
9603        harness.clock.advance_secs(11);
9604        harness.processes.set_alive(false);
9605        harness
9606            .github
9607            .observe(GithubRunnerObservation::NotRegistered);
9608        harness.launcher.supervise(&harness.policy).await.unwrap();
9609        let diagnostic = fs::read_to_string(
9610            harness
9611                .launcher
9612                .diagnostics_root
9613                .join(format!("{}.log", attempt.id)),
9614        )
9615        .unwrap();
9616        assert!(diagnostic.contains("exited_idle_without_work"));
9617        assert!(!diagnostic.contains(JIT));
9618        assert!(!diagnostic.contains("ghp_"));
9619        assert!(!attempt.runtime_path().exists());
9620    }
9621
9622    #[test]
9623    fn native_process_listing_never_contains_jit_and_handoffs_never_survive() {
9624        let root = tempfile::tempdir().unwrap();
9625        let policy = fixtures::policy()
9626            .repository("octo/repo")
9627            .autoscale("home", 1)
9628            .active()
9629            .build();
9630        let runtime = root.path().join("successful");
9631        fs::create_dir_all(&runtime).unwrap();
9632        let processes = NativeProcesses::new();
9633        let config = EncodedJitConfig::new(JIT);
9634        let handoff =
9635            RestrictiveHandoff::create(&runtime, SecretString::from(config.expose().to_owned()))
9636                .unwrap();
9637        let mut child = native_inspection_spec()
9638            .spawn_runner_with_handoff(&handoff)
9639            .expect("native child starts");
9640        let pid = child.pid();
9641        handoff.delete().unwrap();
9642        let command_line = native_command_line(pid);
9643        assert!(
9644            !command_line.contains(JIT),
9645            "the encoded JIT configuration appeared in the native process listing"
9646        );
9647        assert_no_jit_file(&runtime);
9648        child
9649            .stop(Duration::from_secs(1))
9650            .expect("native child stops");
9651
9652        let failed_runtime = root.path().join("failed");
9653        fs::create_dir_all(&failed_runtime).unwrap();
9654        let failed = RunnerAttempt::allocate(
9655            AttemptId::new_random(),
9656            policy.id,
9657            &failed_runtime,
9658            FakeClock::default().now(),
9659        );
9660        assert!(
9661            processes
9662                .spawn(&failed, &EncodedJitConfig::new(JIT))
9663                .is_err(),
9664            "a runtime with no runner executable must fail"
9665        );
9666        assert_no_jit_file(&failed_runtime);
9667        processes
9668            .record_terminate_intent(&failed)
9669            .expect("the intent file and its directory entry are durably synced");
9670        assert_eq!(
9671            fs::read(NativeProcesses::intent_path(&failed)).unwrap(),
9672            b"registration-timeout\n"
9673        );
9674    }
9675
9676    #[test]
9677    fn post_spawn_boundaries_are_bounded_durable_and_never_retry_jit() {
9678        let root = tempfile::tempdir().unwrap();
9679        let policy = fixtures::policy()
9680            .repository("octo/repo")
9681            .autoscale("home", 1)
9682            .active()
9683            .build();
9684        let processes = NativeProcesses::new();
9685        processes.use_long_lived_test_listener();
9686        for (index, boundary) in [
9687            PostSpawnBoundary::HandoffDelete,
9688            PostSpawnBoundary::IdentitySerialize,
9689            PostSpawnBoundary::IdentityWrite,
9690            PostSpawnBoundary::ChildMapInsert,
9691        ]
9692        .into_iter()
9693        .enumerate()
9694        {
9695            let runtime = root.path().join(format!("post-spawn-{index}"));
9696            let bin = runtime.join("bin");
9697            fs::create_dir_all(&bin).unwrap();
9698            #[cfg(windows)]
9699            let listener = bin.join("Runner.Listener.exe");
9700            #[cfg(not(windows))]
9701            let listener = bin.join("Runner.Listener");
9702            fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9703            let attempt = RunnerAttempt::allocate(
9704                AttemptId::new_random(),
9705                policy.id,
9706                &runtime,
9707                FakeClock::default().now(),
9708            );
9709            processes.fail_post_spawn_at(boundary);
9710            let failure = processes
9711                .spawn(&attempt, &EncodedJitConfig::new(JIT))
9712                .expect_err("fault must cross the post-spawn cleanup path");
9713            assert!(!failure.retryable, "{boundary:?} allowed duplicate retry");
9714            assert!(
9715                !processes.is_alive(&attempt).unwrap(),
9716                "{boundary:?} left a child"
9717            );
9718            assert!(!NativeProcesses::identity_path(&attempt).exists());
9719            assert_no_jit_file(&runtime);
9720        }
9721        assert_eq!(processes.post_spawn_reaps.load(Ordering::SeqCst), 4);
9722
9723        let runtime = root.path().join("identity-and-stop-fail");
9724        let bin = runtime.join("bin");
9725        fs::create_dir_all(&bin).unwrap();
9726        #[cfg(windows)]
9727        let listener = bin.join("Runner.Listener.exe");
9728        #[cfg(not(windows))]
9729        let listener = bin.join("Runner.Listener");
9730        fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9731        let attempt = RunnerAttempt::allocate(
9732            AttemptId::new_random(),
9733            policy.id,
9734            &runtime,
9735            FakeClock::default().now(),
9736        );
9737        // The first fault rejects the normal identity write; the second rejects
9738        // its retry after the first stop fails. The fallback sidecar must make
9739        // the live-child result durable without an unbounded reap loop.
9740        processes.fail_post_spawn_at(PostSpawnBoundary::IdentityWrite);
9741        processes.fail_post_spawn_at(PostSpawnBoundary::IdentityWrite);
9742        processes.fail_next_post_spawn_stop();
9743        let failure = processes
9744            .spawn(&attempt, &EncodedJitConfig::new(JIT))
9745            .expect_err("the identity boundary must fail closed");
9746        assert!(failure.live_pid.is_some());
9747        assert_long_lived_listener_ready(&processes, &attempt);
9748        assert!(processes.is_alive(&attempt).unwrap());
9749        assert!(!NativeProcesses::identity_path(&attempt).exists());
9750        assert!(NativeProcesses::fallback_identity_path(&attempt).is_file());
9751        assert_eq!(processes.post_spawn_reaps.load(Ordering::SeqCst), 4);
9752        processes.terminate(&attempt).unwrap();
9753
9754        let runtime = root.path().join("persistent-stop-and-identity-failures");
9755        let bin = runtime.join("bin");
9756        fs::create_dir_all(&bin).unwrap();
9757        #[cfg(windows)]
9758        let listener = bin.join("Runner.Listener.exe");
9759        #[cfg(not(windows))]
9760        let listener = bin.join("Runner.Listener");
9761        fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9762        let mut unresolved = RunnerAttempt::allocate(
9763            AttemptId::new_random(),
9764            policy.id,
9765            &runtime,
9766            FakeClock::default().now(),
9767        );
9768        for _ in 0..3 {
9769            processes.fail_post_spawn_at(PostSpawnBoundary::IdentityWrite);
9770        }
9771        processes.fail_post_spawn_stops(MAX_POST_SPAWN_STOP_ATTEMPTS);
9772        let failure = processes
9773            .spawn(&unresolved, &EncodedJitConfig::new(JIT))
9774            .expect_err("bounded cleanup must return even when every stop errors");
9775        let pid = failure
9776            .live_pid
9777            .expect("the owned child remains supervised in this invocation");
9778        assert!(matches!(failure.reason, FailureReason::Other(_)));
9779        assert_long_lived_listener_ready(&processes, &unresolved);
9780        unresolved.jit_received(FakeClock::default().now()).unwrap();
9781        unresolved.started(pid, FakeClock::default().now()).unwrap();
9782        let journal = SqliteStore::open_in_memory().unwrap();
9783        journal.record_attempt(&unresolved).unwrap();
9784        let recovered = journal.attempt(unresolved.id).unwrap().unwrap();
9785        assert_eq!(recovered.process_id(), Some(pid));
9786        assert_eq!(recovered.state(), AttemptState::Starting);
9787        assert!(processes.is_alive(&unresolved).unwrap());
9788        assert!(!NativeProcesses::identity_path(&unresolved).exists());
9789        assert!(!NativeProcesses::fallback_identity_path(&unresolved).exists());
9790        assert_eq!(
9791            fs::read_to_string(NativeProcesses::unresolved_process_path(&unresolved)).unwrap(),
9792            pid.to_string(),
9793            "bounded cleanup must leave durable unresolved-process evidence before returning"
9794        );
9795        assert!(
9796            NativeProcesses::new().is_alive(&recovered).is_err(),
9797            "restart must fail closed on the durable starting/PID journal rather than trust a bare PID"
9798        );
9799        processes.terminate(&unresolved).unwrap();
9800
9801        let runtime = root.path().join("post-spawn-stop-failed");
9802        let bin = runtime.join("bin");
9803        fs::create_dir_all(&bin).unwrap();
9804        #[cfg(windows)]
9805        let listener = bin.join("Runner.Listener.exe");
9806        #[cfg(not(windows))]
9807        let listener = bin.join("Runner.Listener");
9808        fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9809        let attempt = RunnerAttempt::allocate(
9810            AttemptId::new_random(),
9811            policy.id,
9812            &runtime,
9813            FakeClock::default().now(),
9814        );
9815        processes.fail_post_spawn_at(PostSpawnBoundary::ChildMapInsert);
9816        processes.fail_next_post_spawn_stop();
9817        let failure = processes
9818            .spawn(&attempt, &EncodedJitConfig::new(JIT))
9819            .expect_err("the injected stop failure must preserve supervision");
9820        let live_pid = failure
9821            .live_pid
9822            .expect("live PID is returned to the journal");
9823        assert!(!failure.retryable);
9824        assert_long_lived_listener_ready(&processes, &attempt);
9825        assert!(NativeProcesses::identity_path(&attempt).is_file());
9826        assert_eq!(
9827            NativeProcesses::new()
9828                .read_identity(&attempt)
9829                .unwrap()
9830                .runner()
9831                .unwrap()
9832                .pid(),
9833            live_pid
9834        );
9835        assert_eq!(processes.post_spawn_reaps.load(Ordering::SeqCst), 4);
9836        processes.terminate(&attempt).unwrap();
9837    }
9838
9839    #[test]
9840    #[ignore = "spawned only as the platform-stable native listener fixture"]
9841    fn long_lived_native_listener_helper() {
9842        let ready = std::env::var_os("RUNNER_MANAGER_TEST_LISTENER_READY")
9843            .map(PathBuf::from)
9844            .expect("the parent supplies the readiness path");
9845        // Atomically: the parent polls for the file and must never read it
9846        // created but not yet written.
9847        runner_manager_platform::host_fitness::write_atomically(&ready, b"ready\n")
9848            .expect("the listener publishes readiness");
9849        std::thread::sleep(Duration::from_secs(30));
9850    }
9851
9852    fn assert_long_lived_listener_ready(processes: &NativeProcesses, attempt: &RunnerAttempt) {
9853        let ready = attempt.runtime_path().join(TEST_LISTENER_READY);
9854        let deadline = std::time::Instant::now() + Duration::from_secs(5);
9855        loop {
9856            if ready.is_file() {
9857                assert_eq!(fs::read(&ready).unwrap(), b"ready\n");
9858                return;
9859            }
9860            assert!(
9861                processes.is_alive(attempt).unwrap(),
9862                "the native listener exited before publishing readiness"
9863            );
9864            assert!(
9865                std::time::Instant::now() < deadline,
9866                "the native listener stayed alive but never published readiness"
9867            );
9868            std::thread::sleep(Duration::from_millis(10));
9869        }
9870    }
9871
9872    #[tokio::test]
9873    async fn a_reservation_is_journalled_before_any_effect_and_holds_its_slot() {
9874        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9875        harness.ready().await;
9876
9877        let reserved = harness.reserve().await.unwrap();
9878
9879        assert_eq!(reserved.state(), AttemptState::Allocated);
9880        assert!(reserved.counts_against_capacity());
9881        let journalled = harness.store.attempt(reserved.id).unwrap().unwrap();
9882        assert_eq!(journalled.state(), AttemptState::Allocated);
9883        assert_eq!(harness.packages.materializations.load(Ordering::SeqCst), 0);
9884        assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
9885
9886        let started = harness
9887            .launcher
9888            .finish_launch(&harness.policy, reserved, ReservationWaits::default())
9889            .await
9890            .unwrap();
9891        assert_eq!(started.state(), AttemptState::Starting);
9892    }
9893
9894    /// A launch runs beside its target's pass: the pass does not read a runner
9895    /// still coming up as one that is gone. It has no process and no registration
9896    /// yet, and the bring-up is writing its journal entry.
9897    #[tokio::test]
9898    async fn a_pass_leaves_a_runner_coming_up_alone_and_supervises_it_once_up() {
9899        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9900        harness.ready().await;
9901        let reserved = harness.reserve().await.unwrap();
9902        let looks_left = || harness.github.observations.lock().unwrap().len();
9903        harness
9904            .github
9905            .observations
9906            .lock()
9907            .unwrap()
9908            .push_back(LifecycleGithubObservation::not_registered());
9909
9910        harness.launcher.supervise(&harness.policy).await.unwrap();
9911        assert_eq!(looks_left(), 1, "it did not look");
9912        let journalled = harness.store.attempt(reserved.id).unwrap().unwrap();
9913        assert_eq!(journalled.state(), AttemptState::Allocated);
9914
9915        harness
9916            .launcher
9917            .finish_launch(&harness.policy, reserved, ReservationWaits::default())
9918            .await
9919            .unwrap();
9920        assert!(harness.launcher.bringing_up.lock().unwrap().is_empty());
9921
9922        harness.launcher.supervise(&harness.policy).await.unwrap();
9923        assert_eq!(looks_left(), 0, "up, it is this pass's to supervise");
9924    }
9925
9926    /// A daemon that dies between the reservation and the process: the
9927    /// journalled attempt is what recovery finds, and the slot comes back.
9928    #[tokio::test]
9929    async fn a_crash_between_reservation_and_spawn_recovers_the_slot() {
9930        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9931        harness.ready().await;
9932        let reserved = harness.reserve().await.unwrap();
9933        harness.clock.advance_secs(31);
9934
9935        harness
9936            .launcher
9937            .recover_startup(std::slice::from_ref(&harness.policy))
9938            .await
9939            .unwrap();
9940
9941        let recovered = harness.store.attempt(reserved.id).unwrap().unwrap();
9942        assert!(
9943            !recovered.counts_against_capacity(),
9944            "the slot is free again: {:?}",
9945            recovered.state()
9946        );
9947    }
9948
9949    /// A crash once the package is there, before the process: same result.
9950    #[tokio::test]
9951    async fn a_crash_after_the_package_is_copied_recovers_the_slot() {
9952        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9953        harness.ready().await;
9954        let reserved = harness.reserve().await.unwrap();
9955        harness
9956            .packages
9957            .materialize(&reserved, &mut PackageTimings::default())
9958            .await
9959            .unwrap();
9960        harness.clock.advance_secs(31);
9961
9962        harness
9963            .launcher
9964            .recover_startup(std::slice::from_ref(&harness.policy))
9965            .await
9966            .unwrap();
9967
9968        let recovered = harness.store.attempt(reserved.id).unwrap().unwrap();
9969        assert!(!recovered.counts_against_capacity());
9970    }
9971
9972    #[tokio::test]
9973    async fn runners_prepare_at_once_within_the_bound() {
9974        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9975        harness.ready().await;
9976        let turns = harness.launcher.turns.available_permits();
9977        harness.launcher.turns.forget_permits(turns - 2);
9978        let mut reserved = Vec::new();
9979        for _ in 0..4 {
9980            reserved.push(harness.reserve().await.unwrap());
9981        }
9982
9983        let finished = futures::future::join_all(reserved.into_iter().map(|attempt| {
9984            harness
9985                .launcher
9986                .finish_launch(&harness.policy, attempt, ReservationWaits::default())
9987        }))
9988        .await;
9989
9990        assert!(finished.iter().all(Result::is_ok));
9991        assert_eq!(
9992            harness.packages.peak_materializing.load(Ordering::SeqCst),
9993            2
9994        );
9995    }
9996
9997    #[tokio::test]
9998    async fn every_production_launch_prunes_under_the_same_allocation_guard() {
9999        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
10000        harness.ready().await;
10001        assert_eq!(harness.packages.prunes.load(Ordering::SeqCst), 0);
10002        harness.launch().await;
10003        assert_eq!(harness.packages.prunes.load(Ordering::SeqCst), 1);
10004    }
10005
10006    fn copy_package_tree(
10007        source: &Path,
10008        destination: &Path,
10009        seeds: &TrustedTrees,
10010    ) -> std::io::Result<()> {
10011        let origin = copy_origin(source, destination, seeds);
10012        copy_package_from(source, &origin, destination, seeds)
10013    }
10014
10015    fn assert_no_jit_file(runtime: &Path) {
10016        for entry in fs::read_dir(runtime).unwrap() {
10017            let path = entry.unwrap().path();
10018            if path.is_file() {
10019                let bytes = fs::read(&path).unwrap();
10020                assert!(
10021                    !bytes
10022                        .windows(JIT.len())
10023                        .any(|window| window == JIT.as_bytes()),
10024                    "a JIT payload survived in a runtime file"
10025                );
10026            }
10027        }
10028    }
10029
10030    #[test]
10031    fn production_listener_command_uses_the_supported_jit_contract() {
10032        let runtime = Path::new("runtime");
10033        let spec = runner_listener_spec(
10034            PathBuf::from("Runner.Listener"),
10035            runtime,
10036            Vec::new(),
10037            &RunnerEnv::default(),
10038        );
10039        let arguments: Vec<_> = spec
10040            .arguments()
10041            .iter()
10042            .map(|argument| argument.to_string_lossy().into_owned())
10043            .collect();
10044
10045        assert_eq!(arguments, ["run"]);
10046        assert!(
10047            !arguments
10048                .iter()
10049                .any(|argument| argument == "--jit-config-file"),
10050            "the obsolete file option would be rejected by Runner.Listener 2.336.0"
10051        );
10052    }
10053
10054    /// What a production runner starts with: the platform defaults, the host's
10055    /// `runner.env` over them, the attempt's own temporary directory, and
10056    /// normal scheduling.
10057    #[test]
10058    fn production_listener_layers_runner_env_over_the_platform_defaults() {
10059        let root = tempfile::tempdir().unwrap();
10060        let runtime = root.path().join("attempt");
10061        let host_env =
10062            RunnerEnv::parse("ELECTRON_CACHE=/cache/electron\nHOME=/operator/chose/this\n")
10063                .unwrap();
10064        let caches = vec![
10065            (
10066                "ELECTRON_CACHE",
10067                std::ffi::OsString::from("/rman/_cache/o/r/electron"),
10068            ),
10069            (
10070                "npm_config_cache",
10071                std::ffi::OsString::from("/rman/_cache/o/r/npm"),
10072            ),
10073        ];
10074        let spec = runner_listener_spec(
10075            PathBuf::from("Runner.Listener"),
10076            &runtime,
10077            caches,
10078            &host_env,
10079        );
10080        let env = |name: &str| spec.configured_env(name).map(PathBuf::from);
10081
10082        assert!(spec.expects_normal_scheduling());
10083        assert_eq!(
10084            env("ELECTRON_CACHE"),
10085            Some(PathBuf::from("/cache/electron")),
10086            "runner.env must win over a dependency cache of the same name"
10087        );
10088        assert_eq!(
10089            env("npm_config_cache"),
10090            Some(PathBuf::from("/rman/_cache/o/r/npm"))
10091        );
10092        assert_eq!(
10093            env("HOME"),
10094            Some(PathBuf::from("/operator/chose/this")),
10095            "runner.env must win over a platform default of the same name"
10096        );
10097        for tmp in ["TMPDIR", "TEMP", "TMP"] {
10098            assert_eq!(env(tmp), Some(runtime.join("tmp")), "{tmp}");
10099        }
10100
10101        let home = runtime.join(runner_env::RUNNER_HOME_DIR);
10102        if cfg!(windows) {
10103            // Each runner's own profile, created inside its attempt, so that
10104            // concurrent jobs stop sharing the service account's.
10105            assert_eq!(env("USERPROFILE"), Some(home.clone()));
10106            assert_eq!(env("APPDATA"), Some(home.join(r"AppData\Roaming")));
10107            assert_eq!(env("LOCALAPPDATA"), Some(home.join(r"AppData\Local")));
10108            assert!(home.join(r"AppData\Roaming").is_dir());
10109            assert!(home.join(r"AppData\Local").is_dir());
10110        } else {
10111            // Unix keeps the account's own profile.
10112            assert_eq!(env("USERPROFILE"), None);
10113            assert!(!home.exists());
10114        }
10115    }
10116
10117    /// Every step shell a runner starts inherits PowerShell's telemetry
10118    /// opt-out from the listener, unless the daemon or `runner.env` decided
10119    /// otherwise; see [`runner_env::POWERSHELL_TELEMETRY_OPTOUT`].
10120    #[test]
10121    fn production_listener_opts_powershell_out_of_telemetry() {
10122        let root = tempfile::tempdir().unwrap();
10123        let runtime = root.path().join("attempt");
10124        let listener = |host_env: &RunnerEnv| {
10125            runner_listener_spec(
10126                PathBuf::from("Runner.Listener"),
10127                &runtime,
10128                Vec::new(),
10129                host_env,
10130            )
10131        };
10132        let optout = |spec: &SpawnSpec| {
10133            spec.configured_env(runner_env::POWERSHELL_TELEMETRY_OPTOUT)
10134                .map(OsStr::to_os_string)
10135        };
10136
10137        let spec = listener(&RunnerEnv::default());
10138        if std::env::var_os(runner_env::POWERSHELL_TELEMETRY_OPTOUT)
10139            .is_some_and(|value| !value.is_empty())
10140        {
10141            // The daemon's own value is inherited as it is.
10142            assert_eq!(optout(&spec), None);
10143        } else {
10144            assert_eq!(
10145                optout(&spec),
10146                Some(runner_env::POWERSHELL_TELEMETRY_OPTOUT_VALUE.into())
10147            );
10148        }
10149
10150        let operator = RunnerEnv::parse("POWERSHELL_TELEMETRY_OPTOUT=0\n").unwrap();
10151        assert_eq!(
10152            optout(&listener(&operator)),
10153            Some("0".into()),
10154            "runner.env must win over the telemetry opt-out"
10155        );
10156    }
10157
10158    #[test]
10159    fn an_invalid_runner_env_fails_the_launch_without_echoing_a_value() {
10160        let root = tempfile::tempdir().unwrap();
10161        let path = runner_env::path_in(root.path());
10162        fs::write(&path, "GOOD=1\nghp_notarealtokenbutshapedlikeone\n").unwrap();
10163        let processes = NativeProcesses::new().with_runner_env_file(path.clone());
10164        let failure = processes.host_env().unwrap_err().to_string();
10165        assert!(
10166            failure.contains("runner.env") && failure.contains("line 2"),
10167            "{failure}"
10168        );
10169        assert!(
10170            !failure.contains("ghp_"),
10171            "the failure echoed a value: {failure}"
10172        );
10173
10174        // `prepare` runs before the JIT registration, so a broken file is
10175        // refused there, before GitHub is asked for anything.
10176        let policy = fixtures::policy()
10177            .repository("octo/repo")
10178            .autoscale("home", 1)
10179            .active()
10180            .build();
10181        let attempt = RunnerAttempt::allocate(
10182            AttemptId::new_random(),
10183            policy.id,
10184            root.path(),
10185            FakeClock::default().now(),
10186        );
10187        assert!(processes.prepare(&attempt, &policy).is_err());
10188
10189        fs::write(&path, "GOOD=1\n").unwrap();
10190        assert!(processes.prepare(&attempt, &policy).is_ok());
10191        assert_eq!(processes.host_env().unwrap().get("GOOD"), Some("1"));
10192        assert!(NativeProcesses::new().host_env().unwrap().is_empty());
10193    }
10194
10195    /// The wiring from `prepare` (which sees the policy) to `spawn` (which
10196    /// does not): the policy's repository decides the namespace, and a broken
10197    /// `caches.toml` is refused before GitHub is asked for anything.
10198    #[test]
10199    fn prepare_hands_the_policys_cache_namespace_to_the_launch() {
10200        let root = tempfile::tempdir().unwrap();
10201        let runner_root = root.path().join("rman");
10202        let runtime = runner_root.join("a1");
10203        fs::create_dir_all(runtime.join("bin")).unwrap();
10204        let config = runner_manager_platform::dependency_cache::config_path_in(root.path());
10205        let source_root = runner_root.clone();
10206        let processes = NativeProcesses::new().with_dependency_caches(DependencyCaches::new(
10207            config.clone(),
10208            Arc::new(move || Some(source_root.clone())),
10209            None,
10210        ));
10211        let policy = fixtures::policy()
10212            .repository("Octo/Repo")
10213            .autoscale("home", 1)
10214            .active()
10215            .build();
10216        let attempt = RunnerAttempt::allocate(
10217            AttemptId::new_random(),
10218            policy.id,
10219            &runtime,
10220            FakeClock::default().now(),
10221        );
10222        assert!(
10223            processes
10224                .cache_env(&attempt, None, &RunnerEnv::default())
10225                .is_empty(),
10226            "a launch with no prepared target, such as a bare `spawn`, has no cache"
10227        );
10228        let prepared = processes.prepare(&attempt, &policy).unwrap();
10229        let variables = processes.cache_env(
10230            &attempt,
10231            prepared.cache_target.as_ref(),
10232            &RunnerEnv::default(),
10233        );
10234        let npm = variables
10235            .iter()
10236            .find(|(name, _)| *name == "npm_config_cache")
10237            .map(|(_, value)| PathBuf::from(value));
10238        assert_eq!(
10239            npm,
10240            Some(
10241                runner_root
10242                    .join("_cache")
10243                    .join("octo")
10244                    .join("repo")
10245                    .join("npm")
10246            )
10247        );
10248
10249        fs::write(&config, "[tools]\nnot-a-tool = true\n").unwrap();
10250        let refused = processes
10251            .prepare(&attempt, &policy)
10252            .unwrap_err()
10253            .to_string();
10254        assert!(refused.contains("caches.toml"), "{refused}");
10255    }
10256
10257    #[cfg(windows)]
10258    fn native_inspection_spec() -> SpawnSpec {
10259        SpawnSpec::new("powershell.exe").args([
10260            "-NoProfile",
10261            "-NonInteractive",
10262            "-Command",
10263            "Start-Sleep -Seconds 30",
10264        ])
10265    }
10266
10267    #[cfg(unix)]
10268    fn native_inspection_spec() -> SpawnSpec {
10269        SpawnSpec::new("/bin/sh").args(["-c", "sleep 30"])
10270    }
10271
10272    #[cfg(windows)]
10273    fn native_command_line(pid: u32) -> String {
10274        let output = std::process::Command::new("powershell.exe")
10275            .args([
10276                "-NoProfile",
10277                "-NonInteractive",
10278                "-Command",
10279                &format!("(Get-CimInstance Win32_Process -Filter 'ProcessId = {pid}').CommandLine"),
10280            ])
10281            .output()
10282            .expect("PowerShell can inspect the native child");
10283        assert!(output.status.success(), "native process inspection failed");
10284        String::from_utf8(output.stdout).expect("Windows command lines are Unicode")
10285    }
10286
10287    #[cfg(target_os = "linux")]
10288    fn native_command_line(pid: u32) -> String {
10289        fs::read(format!("/proc/{pid}/cmdline"))
10290            .map(|bytes| String::from_utf8_lossy(&bytes).replace('\0', " "))
10291            .expect("/proc exposes the native child command line")
10292    }
10293
10294    #[cfg(target_os = "macos")]
10295    fn native_command_line(pid: u32) -> String {
10296        let output = std::process::Command::new("ps")
10297            .args(["-o", "command=", "-p", &pid.to_string()])
10298            .output()
10299            .expect("ps can inspect the native child");
10300        assert!(output.status.success(), "native process inspection failed");
10301        String::from_utf8(output.stdout).expect("the command line is UTF-8")
10302    }
10303
10304    // -- c2: persistent slot allocation -------------------------------------
10305
10306    #[tokio::test]
10307    async fn a_persistent_repository_leases_s1_and_journals_it_before_any_github_effect() {
10308        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10309            .with_persistent_workspace(2);
10310        harness.github.watch_journal(Arc::clone(&harness.store));
10311        harness.ready().await;
10312
10313        let attempt = harness.launch().await;
10314
10315        assert_eq!(
10316            attempt.workspace(),
10317            AttemptWorkspace::persistent_slot(nz(1)),
10318            "the lowest free slot is leased"
10319        );
10320        assert_eq!(attempt.runtime_path(), harness.slot_path(1));
10321        assert!(attempt.holds_slot_lease());
10322        // The exact runtime path is journalled, not re-derived later.
10323        assert_eq!(
10324            harness.attempt(attempt.id).runtime_path(),
10325            harness.slot_path(1)
10326        );
10327
10328        // Step 7 of "Slot allocation": the lease exists before GitHub is asked
10329        // for anything, and the runner's work folder stays the relative `_work`
10330        // the slot root is laid out around.
10331        let facts = harness.github.registration_facts();
10332        assert_eq!(facts.len(), 1);
10333        assert_eq!(
10334            facts[0].leased_slots,
10335            vec![1],
10336            "the lease was journalled first"
10337        );
10338        assert_eq!(facts[0].work_folder, DEFAULT_WORK_FOLDER);
10339    }
10340
10341    #[tokio::test]
10342    async fn a_terminal_but_uncleaned_attempt_keeps_its_slot_without_holding_capacity() {
10343        let harness = Harness::new(
10344            FakeGithubLifecycle::default().fail(true),
10345            Arc::new(PersistentDemand),
10346        )
10347        .with_persistent_workspace(2);
10348        harness.ready().await;
10349
10350        // A terminal JIT refusal concludes the attempt without cleaning it.
10351        harness.launch_result().await.unwrap_err();
10352        let first = harness.store.attempts().unwrap().remove(0);
10353        assert_eq!(first.state(), AttemptState::Failed);
10354        assert!(
10355            !first.state().counts_against_capacity(),
10356            "a concluded attempt is invisible to host capacity"
10357        );
10358        assert!(
10359            first.holds_slot_lease(),
10360            "and still owns its directory, so its slot is not free"
10361        );
10362
10363        let second = harness.launch().await;
10364        assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
10365        assert_eq!(second.runtime_path(), harness.slot_path(2));
10366        assert_eq!(
10367            harness
10368                .store
10369                .slot_leases_for_policy(harness.policy.id)
10370                .unwrap()
10371                .len(),
10372            2
10373        );
10374    }
10375
10376    #[tokio::test]
10377    async fn two_sequential_allocations_at_capacity_one_reuse_s1_and_its_retained_work() {
10378        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10379            .with_persistent_workspace(1);
10380        harness.ready().await;
10381
10382        let first = harness.launch().await;
10383        assert_eq!(first.runtime_path(), harness.slot_path(1));
10384
10385        // What a job leaves behind, at the path the runner writes it to.
10386        let checkout = harness.slot_path(1).join(DEFAULT_WORK_FOLDER).join("repo");
10387        fs::create_dir_all(&checkout).unwrap();
10388        fs::write(checkout.join("checkout.txt"), b"from the first job").unwrap();
10389
10390        harness.cleanup_retaining_work(first.id).await;
10391
10392        let second = harness.launch().await;
10393        assert_ne!(second.id, first.id);
10394        assert_eq!(
10395            second.workspace(),
10396            AttemptWorkspace::persistent_slot(nz(1)),
10397            "a released slot is leased again rather than skipped"
10398        );
10399        assert_eq!(
10400            second.runtime_path(),
10401            first.runtime_path(),
10402            "the same slot is the same exact path"
10403        );
10404        assert_eq!(
10405            fs::read_to_string(checkout.join("checkout.txt")).unwrap(),
10406            "from the first job",
10407            "the retained job workspace survived the second allocation"
10408        );
10409        // The attempt's own runner material was recreated for this attempt.
10410        assert!(harness.slot_path(1).join("runner-package").exists());
10411    }
10412
10413    #[tokio::test]
10414    async fn lowering_capacity_leaves_higher_slots_alone_and_raising_it_permits_them_again() {
10415        let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10416            .with_persistent_workspace(2);
10417        harness.ready().await;
10418
10419        let first = harness.launch().await;
10420        let second = harness.launch().await;
10421        assert_eq!(second.runtime_path(), harness.slot_path(2));
10422        let kept = harness
10423            .slot_path(2)
10424            .join(DEFAULT_WORK_FOLDER)
10425            .join("kept.txt");
10426        fs::create_dir_all(kept.parent().unwrap()).unwrap();
10427        fs::write(&kept, b"s2 was here").unwrap();
10428        harness.cleanup_retaining_work(second.id).await;
10429
10430        // The operator lowers the ceiling while s1 is still leased.
10431        harness.policy.set_max_capacity(nz(1)).unwrap();
10432        let refusal = harness.launch_result().await.unwrap_err().to_string();
10433        assert!(
10434            refusal.contains("s1 to s1"),
10435            "the refusal names the ceiling it reached: {refusal}"
10436        );
10437        assert!(
10438            harness.slot_path(2).exists() && kept.exists(),
10439            "lowering capacity deletes nothing; the higher slot is merely unusable"
10440        );
10441
10442        // Raising it again makes the free higher slot available.
10443        harness.policy.set_max_capacity(nz(2)).unwrap();
10444        let third = harness.launch().await;
10445        assert_eq!(third.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
10446        assert_eq!(third.runtime_path(), harness.slot_path(2));
10447        assert_eq!(fs::read_to_string(&kept).unwrap(), "s2 was here");
10448        assert!(first.holds_slot_lease(), "s1 was never disturbed");
10449    }
10450
10451    #[tokio::test]
10452    async fn organization_and_ephemeral_policies_never_enter_slot_allocation() {
10453        for policy in [
10454            fixtures::policy()
10455                .organization("octo")
10456                .autoscale("home", 2)
10457                .active()
10458                .build(),
10459            fixtures::policy()
10460                .repository("octo/repo")
10461                .autoscale("home", 2)
10462                .active()
10463                .build(),
10464        ] {
10465            let mut harness =
10466                Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10467                    .with_host_runner_root();
10468            assert_eq!(policy.workspace_policy(), &WorkspacePolicy::Ephemeral);
10469            harness.policy = policy;
10470            harness.ready().await;
10471
10472            let attempt = harness.launch().await;
10473            assert_eq!(attempt.workspace(), AttemptWorkspace::Ephemeral);
10474            assert_eq!(attempt.workspace().slot_number(), None);
10475            assert!(!attempt.holds_slot_lease());
10476            assert_eq!(
10477                attempt.runtime_path().parent().unwrap(),
10478                harness.host_root(),
10479                "a disposable attempt is a child of the host root, never of a slot"
10480            );
10481            assert!(
10482                harness
10483                    .store
10484                    .slot_leases_for_policy(harness.policy.id)
10485                    .unwrap()
10486                    .is_empty()
10487            );
10488        }
10489    }
10490
10491    #[tokio::test]
10492    async fn two_concurrent_allocations_never_share_a_slot() {
10493        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10494            .with_persistent_workspace(2);
10495        harness.github.watch_journal(Arc::clone(&harness.store));
10496        harness.ready().await;
10497
10498        // Both allocators race for the same host allocation lock, which is what
10499        // orders slot *selection*; each one then journals its lease before it
10500        // asks GitHub for anything.
10501        let (first, second) = tokio::join!(harness.launch_result(), harness.launch_result());
10502        let first = first.unwrap();
10503        let second = second.unwrap();
10504
10505        let slots: BTreeSet<u16> = [&first, &second]
10506            .iter()
10507            .map(|attempt| {
10508                attempt
10509                    .workspace()
10510                    .slot_number()
10511                    .expect("a persistent attempt leases a slot")
10512            })
10513            .collect();
10514        assert_eq!(slots, BTreeSet::from([1, 2]), "one slot each, never shared");
10515        assert_ne!(first.runtime_path(), second.runtime_path());
10516        assert_eq!(
10517            harness
10518                .store
10519                .slot_leases_for_policy(harness.policy.id)
10520                .unwrap()
10521                .len(),
10522            2
10523        );
10524
10525        // Every registration saw *its own* lease already in the journal. Reading
10526        // the whole journal and asking only that it be non-empty would pass on
10527        // the other allocator's lease, which is precisely the ordering bug this
10528        // test exists to exclude.
10529        let facts = harness.github.registration_facts();
10530        assert_eq!(facts.len(), 2);
10531        for fact in facts {
10532            let attempt = [&first, &second]
10533                .into_iter()
10534                .find(|attempt| runner_name(attempt.id) == fact.runner_name)
10535                .expect("every registration belongs to one of the two attempts");
10536            let slot = attempt
10537                .workspace()
10538                .slot_number()
10539                .expect("a persistent attempt leases a slot");
10540            assert!(
10541                fact.leased_slots.contains(&slot),
10542                "a JIT request never precedes its own lease: s{slot} not in {:?}",
10543                fact.leased_slots
10544            );
10545            assert_eq!(fact.work_folder, DEFAULT_WORK_FOLDER);
10546        }
10547    }
10548
10549    #[tokio::test]
10550    async fn the_database_is_the_final_fence_against_two_attempts_in_one_slot() {
10551        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10552            .with_persistent_workspace(2);
10553        harness.ready().await;
10554        let first = harness.launch().await;
10555
10556        // What a second allocator that lost the race would write: the lock
10557        // orders selection, and this index is what catches a writer the lock
10558        // could not see.
10559        let clash = RunnerAttempt::allocate_in(
10560            AttemptId::new_random(),
10561            harness.policy.id,
10562            first.runtime_path(),
10563            AttemptWorkspace::persistent_slot(nz(1)),
10564            harness.clock.now(),
10565        );
10566        assert!(matches!(
10567            harness.store.record_attempt(&clash).unwrap_err(),
10568            StoreError::SlotAlreadyLeased { slot: 1, .. }
10569        ));
10570
10571        // And the launcher reports it as itself rather than as a generic
10572        // journal failure, so the operator reads what actually happened.
10573        let error = harness.launcher.record_allocation(&clash).unwrap_err();
10574        let rendered = error.to_string();
10575        assert!(rendered.contains("slot s1"), "{rendered}");
10576        assert!(rendered.contains("nothing was written"), "{rendered}");
10577        assert_eq!(
10578            harness.store.attempts().unwrap().len(),
10579            1,
10580            "the losing allocator journalled nothing"
10581        );
10582    }
10583
10584    #[test]
10585    fn slot_selection_fills_the_lowest_gap_and_stops_at_the_ceiling() {
10586        let leased = |slots: &[u16]| -> Vec<RunnerAttempt> {
10587            slots
10588                .iter()
10589                .map(|slot| {
10590                    RunnerAttempt::allocate_in(
10591                        AttemptId::new_random(),
10592                        fixtures::POLICY_ID,
10593                        format!("/srv/rman/acme/s{slot}"),
10594                        AttemptWorkspace::persistent_slot(nz(*slot)),
10595                        fixtures::created_at(),
10596                    )
10597                })
10598                .collect()
10599        };
10600
10601        assert_eq!(lowest_free_slot(&[], nz(1)), Some(nz(1)));
10602        assert_eq!(lowest_free_slot(&leased(&[1]), nz(4)), Some(nz(2)));
10603        // The gap a released middle slot leaves is filled before the tail.
10604        assert_eq!(lowest_free_slot(&leased(&[1, 3]), nz(4)), Some(nz(2)));
10605        // The ceiling is a refusal, never a reason to allocate past it.
10606        assert_eq!(lowest_free_slot(&leased(&[1]), nz(1)), None);
10607        assert_eq!(lowest_free_slot(&leased(&[1, 2]), nz(2)), None);
10608        // An ephemeral attempt holds no slot and cannot block one.
10609        let ephemeral = vec![RunnerAttempt::allocate(
10610            AttemptId::new_random(),
10611            fixtures::POLICY_ID,
10612            "/srv/rman/host/abc",
10613            fixtures::created_at(),
10614        )];
10615        assert_eq!(lowest_free_slot(&ephemeral, nz(1)), Some(nz(1)));
10616    }
10617
10618    #[test]
10619    fn a_slot_is_reusable_only_when_it_is_empty_or_holds_one_real_work_directory() {
10620        let root = tempfile::tempdir().unwrap();
10621        let slot = root.path().join("s1");
10622        fs::create_dir(&slot).unwrap();
10623        accept_reusable_slot(&slot).expect("an empty slot is reusable");
10624
10625        fs::create_dir(slot.join(DEFAULT_WORK_FOLDER)).unwrap();
10626        accept_reusable_slot(&slot).expect("a retained job workspace is reusable");
10627
10628        // Runner material a previous attempt left behind is refused rather than
10629        // reused or removed: deciding those bytes are safe is cleanup's job.
10630        fs::create_dir(slot.join("bin")).unwrap();
10631        fs::write(slot.join(".github-runner-id"), b"73").unwrap();
10632        let refusal = accept_reusable_slot(&slot).unwrap_err().to_string();
10633        assert!(refusal.contains("bin"), "{refusal}");
10634        assert!(refusal.contains(".github-runner-id"), "{refusal}");
10635
10636        // A `_work` that is not a real directory is not a job workspace.
10637        let file_work = root.path().join("s2");
10638        fs::create_dir(&file_work).unwrap();
10639        fs::write(file_work.join(DEFAULT_WORK_FOLDER), b"not a directory").unwrap();
10640        assert!(accept_reusable_slot(&file_work).is_err());
10641    }
10642
10643    #[cfg(unix)]
10644    #[test]
10645    fn a_link_shaped_work_directory_is_refused_rather_than_followed() {
10646        // Windows needs a privilege to create either kind of link, so the
10647        // link-shaped cases are asserted here; the rule itself is
10648        // platform-independent because it is `symlink_metadata`'s answer.
10649        let root = tempfile::tempdir().unwrap();
10650        let elsewhere = root.path().join("elsewhere");
10651        fs::create_dir(&elsewhere).unwrap();
10652
10653        let slot = root.path().join("s1");
10654        fs::create_dir(&slot).unwrap();
10655        std::os::unix::fs::symlink(&elsewhere, slot.join(DEFAULT_WORK_FOLDER)).unwrap();
10656        assert!(accept_reusable_slot(&slot).is_err());
10657
10658        let linked_slot = root.path().join("s2");
10659        std::os::unix::fs::symlink(&elsewhere, &linked_slot).unwrap();
10660        assert!(create_or_validate_slot(&linked_slot).is_err());
10661    }
10662
10663    #[test]
10664    fn a_slot_standing_where_a_file_is_refuses_rather_than_replacing_it() {
10665        let root = tempfile::tempdir().unwrap();
10666        let occupied = root.path().join("s1");
10667        fs::write(&occupied, b"an operator's file").unwrap();
10668        let refusal = create_or_validate_slot(&occupied).unwrap_err().to_string();
10669        assert!(refusal.contains("is not a directory"), "{refusal}");
10670        assert_eq!(fs::read_to_string(&occupied).unwrap(), "an operator's file");
10671
10672        let fresh = root.path().join("s2");
10673        create_or_validate_slot(&fresh).expect("a missing slot is created");
10674        assert!(fresh.is_dir());
10675        create_or_validate_slot(&fresh).expect("an existing directory is accepted");
10676    }
10677
10678    #[test]
10679    fn the_retained_work_directory_is_matched_the_way_the_filesystem_matches_it() {
10680        assert!(is_work_folder(OsStr::new(DEFAULT_WORK_FOLDER)));
10681        assert!(!is_work_folder(OsStr::new("_work2")));
10682        // A Windows filesystem is case-insensitive, so `_Work` *is* the retained
10683        // job workspace there and must never be removed as a leftover; on a
10684        // case-sensitive filesystem it is a different directory entirely.
10685        assert_eq!(is_work_folder(OsStr::new("_Work")), cfg!(windows));
10686    }
10687
10688    #[test]
10689    fn package_materialization_never_overwrites_or_follows_a_retained_work_directory() {
10690        let root = tempfile::tempdir().unwrap();
10691        let package = root.path().join("package");
10692        fs::create_dir_all(package.join("bin")).unwrap();
10693        fs::write(package.join("bin").join("Runner.Listener"), b"binary").unwrap();
10694        // A nested `_work` inside the package's own tree is an ordinary name.
10695        fs::create_dir_all(package.join("externals").join(DEFAULT_WORK_FOLDER)).unwrap();
10696
10697        let slot = root.path().join("s1");
10698        let retained = slot.join(DEFAULT_WORK_FOLDER).join("repo");
10699        fs::create_dir_all(&retained).unwrap();
10700        fs::write(retained.join("checkout.txt"), b"from the first job").unwrap();
10701
10702        copy_package_tree(&package, &slot, &TrustedTrees::default())
10703            .expect("the package lays out around `_work`");
10704        assert!(slot.join("bin").join("Runner.Listener").exists());
10705        assert!(
10706            slot.join("externals").join(DEFAULT_WORK_FOLDER).is_dir(),
10707            "the guard is top-level only"
10708        );
10709        assert_eq!(
10710            fs::read_to_string(retained.join("checkout.txt")).unwrap(),
10711            "from the first job"
10712        );
10713
10714        // A package that ever grew a top-level `_work` is refused, not merged.
10715        fs::create_dir(package.join(DEFAULT_WORK_FOLDER)).unwrap();
10716        let error = copy_package_tree(&package, &slot, &TrustedTrees::default()).unwrap_err();
10717        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
10718        assert_eq!(
10719            fs::read_to_string(retained.join("checkout.txt")).unwrap(),
10720            "from the first job"
10721        );
10722    }
10723
10724    #[test]
10725    fn rolling_back_a_materialization_keeps_a_slot_but_removes_a_disposable_directory() {
10726        let root = tempfile::tempdir().unwrap();
10727
10728        let slot = root.path().join("s1");
10729        let retained = slot.join(DEFAULT_WORK_FOLDER);
10730        fs::create_dir_all(retained.join("repo")).unwrap();
10731        fs::write(retained.join("repo").join("checkout.txt"), b"kept").unwrap();
10732        fs::create_dir_all(slot.join("bin")).unwrap();
10733        fs::write(slot.join(".github-runner-id"), b"73").unwrap();
10734        let persistent = RunnerAttempt::allocate_in(
10735            AttemptId::new_random(),
10736            fixtures::POLICY_ID,
10737            &slot,
10738            AttemptWorkspace::persistent_slot(nz(1)),
10739            fixtures::created_at(),
10740        );
10741
10742        remove_materialized_package(&persistent).unwrap();
10743        assert!(slot.is_dir(), "the slot itself is not removed");
10744        assert!(!slot.join("bin").exists());
10745        assert!(!slot.join(".github-runner-id").exists());
10746        assert_eq!(
10747            fs::read_to_string(retained.join("repo").join("checkout.txt")).unwrap(),
10748            "kept"
10749        );
10750
10751        let disposable_path = root.path().join("abcdef012345");
10752        fs::create_dir_all(disposable_path.join(DEFAULT_WORK_FOLDER)).unwrap();
10753        let disposable = RunnerAttempt::allocate(
10754            AttemptId::new_random(),
10755            fixtures::POLICY_ID,
10756            &disposable_path,
10757            fixtures::created_at(),
10758        );
10759        remove_materialized_package(&disposable).unwrap();
10760        assert!(
10761            !disposable_path.exists(),
10762            "a disposable directory is still removed whole"
10763        );
10764    }
10765
10766    // -- c3: persistent cleanup and recovery --------------------------------
10767
10768    /// The runner state one attempt leaves at a slot root, as a real attempt
10769    /// leaves it: binaries, registration identity, a JIT handoff that outlived
10770    /// its process, and this agent's own lifecycle sidecars.
10771    ///
10772    /// Driven by [`SENSITIVE_SLOT_ENTRIES`] rather than by a second copy of it,
10773    /// so a name added to the thing cleanup must prove absent is a name every
10774    /// test here starts leaving behind.
10775    fn litter_the_slot(slot: &Path) {
10776        for directory in ["bin", "externals", "_diag"] {
10777            fs::create_dir_all(slot.join(directory)).unwrap();
10778        }
10779        fs::write(slot.join("bin").join("Runner.Listener"), b"binary").unwrap();
10780        for file in SENSITIVE_SLOT_ENTRIES
10781            .iter()
10782            .filter(|entry| !slot.join(entry).is_dir())
10783        {
10784            fs::write(slot.join(file), b"runner state").unwrap();
10785        }
10786        // A handoff whose owning process died before `Drop` could delete it.
10787        fs::write(
10788            slot.join(format!(
10789                "{}0f1e2d3c-4b5a-6978-8796-a5b4c3d2e1f0.tmp",
10790                RestrictiveHandoff::NAME_PREFIX
10791            )),
10792            JIT.as_bytes(),
10793        )
10794        .unwrap();
10795    }
10796
10797    /// A marker under `_work` of the kind a job leaves for the next one.
10798    fn retain_under_work(slot: &Path) -> PathBuf {
10799        let checkout = slot.join(DEFAULT_WORK_FOLDER).join("repo").join("target");
10800        fs::create_dir_all(&checkout).unwrap();
10801        let marker = checkout.join("build-output.bin");
10802        fs::write(&marker, RETAINED).unwrap();
10803        marker
10804    }
10805
10806    /// What a job leaves under `_work` for the next job to reuse.
10807    const RETAINED: &str = "a Git-ignored build output the next job reuses";
10808
10809    /// Every direct entry of a directory, sorted, as plain strings.
10810    fn entries_of(directory: &Path) -> Vec<String> {
10811        let mut names: Vec<String> = fs::read_dir(directory)
10812            .unwrap()
10813            .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
10814            .collect();
10815        names.sort();
10816        names
10817    }
10818
10819    /// The one entry a cleaned slot is allowed to hold.
10820    fn only_the_job_workspace() -> Vec<String> {
10821        vec![DEFAULT_WORK_FOLDER.to_owned()]
10822    }
10823
10824    #[cfg(unix)]
10825    #[test]
10826    fn disposable_tree_removal_does_not_open_a_dotnet_diagnostic_fifo() {
10827        use std::sync::mpsc;
10828
10829        let temporary = tempfile::tempdir().unwrap();
10830        let tree = temporary.path().join("attempt");
10831        let diagnostic = tree.join("tmp/clr-debug-pipe-runner-in");
10832        fs::create_dir_all(diagnostic.parent().unwrap()).unwrap();
10833        assert!(
10834            std::process::Command::new("mkfifo")
10835                .arg(&diagnostic)
10836                .status()
10837                .unwrap()
10838                .success()
10839        );
10840
10841        let (finished, result) = mpsc::channel();
10842        std::thread::spawn(move || {
10843            let removed = remove_runtime_tree(&tree);
10844            let _ = finished.send(removed);
10845        });
10846
10847        result
10848            .recv_timeout(Duration::from_secs(2))
10849            .expect("runtime deletion must not wait for a FIFO peer")
10850            .unwrap();
10851        assert!(!diagnostic.exists());
10852    }
10853
10854    /// The junction WinINet leaves in a job's profile, reproduced: its own DACL
10855    /// denies `Everyone` the right to list it.
10856    #[cfg(windows)]
10857    #[test]
10858    fn disposable_tree_removal_unlinks_a_junction_nobody_may_list() {
10859        let temporary = tempfile::tempdir().unwrap();
10860        let tree = temporary.path().join("attempt");
10861        let cache = tree.join(r"tmp\home\AppData\Local\Microsoft\Windows\INetCache");
10862        let outside = temporary.path().join("outside");
10863        fs::create_dir_all(&cache).unwrap();
10864        fs::create_dir_all(&outside).unwrap();
10865        fs::write(outside.join("keep.txt"), b"not the attempt's").unwrap();
10866        fs::write(tree.join("tmp").join("job.log"), b"job output").unwrap();
10867        let junction = cache.join("Content.IE5");
10868        let created = std::process::Command::new("cmd")
10869            .arg("/C")
10870            .arg("mklink")
10871            .arg("/J")
10872            .arg(&junction)
10873            .arg(&outside)
10874            .output()
10875            .unwrap();
10876        assert!(created.status.success(), "{created:?}");
10877        // `/L` puts the entry on the junction itself, as WinINet does, rather
10878        // than on the directory it points at. `*S-1-1-0` is `Everyone` in every
10879        // display language.
10880        let denied = std::process::Command::new("icacls")
10881            .arg(&junction)
10882            .args(["/L", "/deny", "*S-1-1-0:(RD)"])
10883            .output()
10884            .unwrap();
10885        assert!(denied.status.success(), "{denied:?}");
10886
10887        // The control: without the fix the tree cannot be removed at all.
10888        let unfixed = remove_dir_all::remove_dir_all(&tree)
10889            .expect_err("the fixture no longer reproduces the WinINet junction");
10890        assert_eq!(unfixed.kind(), std::io::ErrorKind::PermissionDenied);
10891        assert!(junction.symlink_metadata().is_ok());
10892
10893        remove_runtime_tree(&tree).expect("a tree holding a deny-list junction is removed");
10894        assert!(!tree.exists(), "the attempt tree survived");
10895        assert_eq!(
10896            fs::read(outside.join("keep.txt")).unwrap(),
10897            b"not the attempt's",
10898            "removing the junction followed it into the directory it points at"
10899        );
10900    }
10901
10902    /// One slot entry that refuses to be removed, and the undo that lets the
10903    /// temporary directory be torn down afterwards.
10904    ///
10905    /// The two operating systems refuse for different reasons and there is no
10906    /// portable third. Windows will not open a file for deletion while a handle
10907    /// with share mode zero is held on it; Unix will not unlink from a directory
10908    /// the caller cannot write. Both are states a real machine reaches -- a
10909    /// scanner holding a file open, a job that left a directory read-only -- so
10910    /// the injection is a filesystem fact rather than a seam cut into the
10911    /// product for a test to pull.
10912    ///
10913    /// The Unix half is a permission, and permissions do not apply to `root`.
10914    /// [`Self::inject`] proves the block on a throwaway directory before
10915    /// claiming it, so a suite running as `root` says it could not inject rather
10916    /// than asserting nothing and passing.
10917    struct BlockedDeletion {
10918        directory: PathBuf,
10919        #[cfg(windows)]
10920        _handle: fs::File,
10921    }
10922
10923    impl BlockedDeletion {
10924        const HELD: &'static str = "held-open";
10925
10926        /// Fill `directory` with a file that cannot be removed, or answer `None`
10927        /// when this account cannot be stopped from removing anything.
10928        fn inject(directory: &Path) -> Option<Self> {
10929            #[cfg(unix)]
10930            if !Self::refusal_is_possible() {
10931                return None;
10932            }
10933            fs::create_dir_all(directory).unwrap();
10934            fs::write(
10935                directory.join(Self::HELD),
10936                b"a file the scrub cannot remove",
10937            )
10938            .unwrap();
10939            #[cfg(windows)]
10940            let handle = {
10941                use std::os::windows::fs::OpenOptionsExt;
10942
10943                fs::OpenOptions::new()
10944                    .read(true)
10945                    .share_mode(0)
10946                    .open(directory.join(Self::HELD))
10947                    .expect("the blocking handle opens")
10948            };
10949            #[cfg(unix)]
10950            Self::set_mode(directory, 0o555);
10951            Some(Self {
10952                directory: directory.to_path_buf(),
10953                #[cfg(windows)]
10954                _handle: handle,
10955            })
10956        }
10957
10958        fn release(self) {
10959            drop(self);
10960        }
10961
10962        #[cfg(unix)]
10963        fn refusal_is_possible() -> bool {
10964            let probe = tempfile::tempdir().unwrap();
10965            let directory = probe.path().join("probe");
10966            fs::create_dir(&directory).unwrap();
10967            fs::write(directory.join("file"), b"probe").unwrap();
10968            Self::set_mode(&directory, 0o555);
10969            let refused = fs::remove_dir_all(&directory).is_err();
10970            Self::set_mode(&directory, 0o755);
10971            refused
10972        }
10973
10974        #[cfg(unix)]
10975        fn set_mode(directory: &Path, mode: u32) {
10976            use std::os::unix::fs::PermissionsExt;
10977
10978            let mut permissions = fs::metadata(directory).unwrap().permissions();
10979            permissions.set_mode(mode);
10980            fs::set_permissions(directory, permissions).unwrap();
10981        }
10982    }
10983
10984    impl Drop for BlockedDeletion {
10985        fn drop(&mut self) {
10986            #[cfg(unix)]
10987            Self::set_mode(&self.directory, 0o755);
10988            #[cfg(not(unix))]
10989            let _ = &self.directory;
10990        }
10991    }
10992
10993    #[tokio::test]
10994    async fn two_sequential_jobs_keep_the_checkout_and_start_without_the_earlier_runner_state() {
10995        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10996            .with_persistent_workspace(1);
10997        harness.ready().await;
10998
10999        let first = harness.launch().await;
11000        let slot = harness.slot_path(1);
11001        assert_eq!(first.runtime_path(), slot);
11002        assert_eq!(
11003            read_runner_id(&slot),
11004            Some(73),
11005            "the attempt registered, so its identity is on disk"
11006        );
11007        let marker = retain_under_work(&slot);
11008        litter_the_slot(&slot);
11009
11010        harness.cleanup_retaining_work(first.id).await;
11011
11012        // The allowlist is exactly one entry, so this assertion is the security
11013        // property in full: what is retained, and that nothing else is.
11014        assert_eq!(entries_of(&slot), only_the_job_workspace());
11015        assert_eq!(fs::read_to_string(&marker).unwrap(), RETAINED);
11016        assert_eq!(
11017            read_runner_id(&slot),
11018            None,
11019            "the first attempt's registration identity is gone before the second starts"
11020        );
11021        assert_eq!(harness.attempt(first.id).state(), AttemptState::Cleaned);
11022        assert!(!harness.attempt(first.id).holds_slot_lease());
11023
11024        let second = harness.launch().await;
11025        assert_ne!(second.id, first.id);
11026        assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(1)));
11027        assert_eq!(
11028            second.runtime_path(),
11029            slot,
11030            "the same slot, so the same retained `_work`"
11031        );
11032        assert_eq!(fs::read_to_string(&marker).unwrap(), RETAINED);
11033    }
11034
11035    #[tokio::test]
11036    async fn cleaning_a_persistent_slot_needs_no_policy_and_scans_no_directory_for_ownership() {
11037        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11038            .with_persistent_workspace(1);
11039        harness.ready().await;
11040        let attempt = harness.launch().await;
11041        let slot = harness.slot_path(1);
11042        let marker = retain_under_work(&slot);
11043        litter_the_slot(&slot);
11044        harness.conclude(attempt.id);
11045
11046        // A repository removed from the product between the attempt concluding
11047        // and the sweep reaching it. The journalled runtime path and slot are
11048        // the only facts left, and `04-security-recovery.md` requires them to be
11049        // enough: the alternative is scanning a root to work out which
11050        // directories were ours, which invariant 6 forbids.
11051        harness
11052            .store
11053            .remove_policy(harness.policy.id, harness.policy.revision())
11054            .unwrap();
11055        assert!(harness.store.policy(harness.policy.id).unwrap().is_none());
11056
11057        harness
11058            .launcher
11059            .clean(attempt.id)
11060            .await
11061            .expect("journal facts alone are enough to clean the slot");
11062
11063        assert_eq!(entries_of(&slot), only_the_job_workspace());
11064        assert!(marker.exists());
11065        assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
11066    }
11067
11068    #[tokio::test]
11069    async fn an_injected_partial_deletion_quarantines_the_slot_across_a_restart() {
11070        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11071            .with_persistent_workspace(2);
11072        harness.ready().await;
11073        let first = harness.launch().await;
11074        let slot = harness.slot_path(1);
11075        let marker = retain_under_work(&slot);
11076        litter_the_slot(&slot);
11077        harness.conclude(first.id);
11078
11079        let Some(block) = BlockedDeletion::inject(&slot.join("bin")) else {
11080            eprintln!(
11081                "skipped: this account cannot be refused a deletion, so no partial deletion can \
11082                 be injected"
11083            );
11084            return;
11085        };
11086
11087        let refusal = harness
11088            .launcher
11089            .clean(first.id)
11090            .await
11091            .expect_err("a deletion that failed may not report a cleaned slot");
11092        let rendered = refusal.reason.to_string();
11093        assert!(rendered.contains("could not be removed"), "{rendered}");
11094
11095        let held = harness.attempt(first.id);
11096        assert_eq!(held.state(), AttemptState::Failed, "still not cleaned");
11097        assert!(held.holds_slot_lease(), "so the slot is still leased");
11098        assert!(
11099            !held.state().counts_against_capacity(),
11100            "and a concluded attempt still costs the host no capacity"
11101        );
11102
11103        // The same journal and the same directories, under a launcher that
11104        // remembers nothing. Recovery must complete: a host that can launch
11105        // nothing at all because one slot is stuck is not what "does not count
11106        // as active host capacity" means.
11107        let restarted = harness.restart();
11108        restarted
11109            .recover_startup(std::slice::from_ref(&harness.policy))
11110            .await
11111            .expect("one quarantined slot does not stop the host recovering");
11112        assert_eq!(
11113            harness.attempt(first.id).state(),
11114            AttemptState::Failed,
11115            "the quarantine survived the restart"
11116        );
11117        assert!(
11118            harness
11119                .reconcile_events
11120                .events()
11121                .iter()
11122                .any(|event| matches!(
11123                    event,
11124                    LifecycleEvent::AttemptCleanFailed {
11125                        reason: "slot_entry_could_not_be_removed",
11126                        ..
11127                    }
11128                )),
11129            "the refusal is reported rather than retried in silence"
11130        );
11131
11132        // Capacity two, slot one quarantined: the next attempt goes to s2 and
11133        // never to the slot still holding runner state.
11134        let second = harness
11135            .launch_through(&restarted)
11136            .await
11137            .expect("the host can still launch");
11138        assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
11139
11140        // And the same cleanup succeeds once the obstruction is gone, which is
11141        // what "retry through normal recovery" has to mean.
11142        block.release();
11143        restarted
11144            .clean(first.id)
11145            .await
11146            .expect("the retried cleanup completes");
11147        assert_eq!(entries_of(&slot), only_the_job_workspace());
11148        assert!(marker.exists());
11149        assert_eq!(harness.attempt(first.id).state(), AttemptState::Cleaned);
11150    }
11151
11152    /// The disposable half of the same injection, which had no test of its own.
11153    ///
11154    /// `scrub_workspace`'s ephemeral arm turns a failed `remove_dir_all` into
11155    /// `"attempt workspace could not be removed"`, and until now that branch was
11156    /// only reachable in theory: every injected-deletion test drove a persistent
11157    /// slot. The property is the same one and matters for the same reason --
11158    /// `04-security-recovery.md`'s "Cleanup partly fails and the slot is reused
11159    /// anyway" -- but the disposable guarantee is stronger, so a cleanup that
11160    /// reported success over a directory it had not removed would be the
11161    /// contamination gate failing silently rather than a slot being held.
11162    #[tokio::test]
11163    async fn an_injected_deletion_failure_leaves_a_disposable_attempt_uncleaned() {
11164        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
11165        harness.ready().await;
11166        let attempt = harness.launch().await;
11167        let runtime = attempt.runtime_path().to_path_buf();
11168        assert_eq!(attempt.workspace(), AttemptWorkspace::Ephemeral);
11169        harness.conclude(attempt.id);
11170
11171        let Some(block) = BlockedDeletion::inject(&runtime.join("held-open-subdirectory")) else {
11172            eprintln!(
11173                "skipped: this account cannot be refused a deletion, so no partial deletion can be injected"
11174            );
11175            return;
11176        };
11177
11178        let refusal = harness
11179            .launcher
11180            .clean(attempt.id)
11181            .await
11182            .expect_err("a deletion that failed may not report a removed workspace");
11183        let rendered = refusal.reason.to_string();
11184        assert!(
11185            rendered.contains("could not be removed"),
11186            "the refusal names what happened: {rendered}"
11187        );
11188        assert_ne!(
11189            harness.attempt(attempt.id).state(),
11190            AttemptState::Cleaned,
11191            "an attempt whose directory is still on disk is not cleaned"
11192        );
11193        assert!(
11194            runtime.is_dir(),
11195            "the directory the removal could not finish is still there, which is the fact the journal must keep agreeing with"
11196        );
11197
11198        let restarted = harness.restart();
11199        restarted
11200            .recover_startup(std::slice::from_ref(&harness.policy))
11201            .await
11202            .expect("one locked ephemeral workspace does not stop startup recovery");
11203        assert!(
11204            harness
11205                .reconcile_events
11206                .events()
11207                .iter()
11208                .any(|event| matches!(
11209                    event,
11210                    LifecycleEvent::AttemptCleanFailed {
11211                        attempt: failed_attempt,
11212                        reason: "ephemeral_workspace_could_not_be_removed",
11213                        ..
11214                    } if *failed_attempt == attempt.id
11215                )),
11216            "the deferred cleanup remains visible while the daemon keeps running"
11217        );
11218
11219        // And the ordinary retry -- the reconciler's next terminal sweep --
11220        // finishes it once the obstruction is gone.
11221        block.release();
11222        harness
11223            .launcher
11224            .clean(attempt.id)
11225            .await
11226            .expect("the retried cleanup completes");
11227        assert!(!runtime.exists(), "the whole attempt directory goes");
11228        assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
11229    }
11230
11231    /// Supervision retries a workspace it cannot remove with per-attempt
11232    /// backoff, so a stuck workspace is reported once per delay rather than on
11233    /// every pass.
11234    #[tokio::test]
11235    async fn supervision_backs_off_a_workspace_it_cannot_remove() {
11236        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
11237        harness.ready().await;
11238        let attempt = harness.launch().await;
11239        let runtime = attempt.runtime_path().to_path_buf();
11240        harness.conclude(attempt.id);
11241        let Some(block) = BlockedDeletion::inject(&runtime.join("held-open-subdirectory")) else {
11242            eprintln!(
11243                "skipped: this account cannot be refused a deletion, so no partial deletion can be injected"
11244            );
11245            return;
11246        };
11247        let failures = || {
11248            harness
11249                .reconcile_events
11250                .events()
11251                .iter()
11252                .filter(|event| {
11253                    matches!(
11254                        event,
11255                        LifecycleEvent::AttemptCleanFailed { attempt: failed, .. }
11256                            if *failed == attempt.id
11257                    )
11258                })
11259                .count()
11260        };
11261        let supervise = async || {
11262            harness
11263                .launcher
11264                .supervise(&harness.policy)
11265                .await
11266                .expect("a stuck workspace does not fail supervision");
11267        };
11268
11269        supervise().await;
11270        assert_eq!(failures(), 1);
11271        supervise().await;
11272        assert_eq!(
11273            failures(),
11274            1,
11275            "the failed cleanup was retried on the very next pass"
11276        );
11277        harness.clock.advance_secs(31);
11278        supervise().await;
11279        assert_eq!(
11280            failures(),
11281            2,
11282            "the cleanup was not retried once its delay passed"
11283        );
11284
11285        block.release();
11286        harness.clock.advance_secs(61);
11287        supervise().await;
11288        assert!(
11289            !runtime.exists(),
11290            "the retry removes the workspace once it can"
11291        );
11292        assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
11293    }
11294
11295    #[tokio::test]
11296    async fn changing_a_repository_back_to_ephemeral_leaves_every_old_slot_untouched() {
11297        let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11298            .with_persistent_workspace(1);
11299        harness.ready().await;
11300        let first = harness.launch().await;
11301        let slot = harness.slot_path(1);
11302        let marker = retain_under_work(&slot);
11303        harness.cleanup_retaining_work(first.id).await;
11304
11305        // Every attempt for this policy is cleaned, so the mutation is allowed
11306        // (`04-security-recovery.md`, "Recovery rules"). What it must not do is
11307        // move or delete anything the operator still owns.
11308        harness
11309            .policy
11310            .set_workspace_policy(WorkspacePolicy::Ephemeral)
11311            .unwrap();
11312
11313        let second = harness.launch().await;
11314        assert_eq!(second.workspace(), AttemptWorkspace::Ephemeral);
11315        assert_eq!(
11316            second.runtime_path().parent().unwrap(),
11317            harness.host_root(),
11318            "a disposable attempt is a child of the host root"
11319        );
11320        assert!(slot.is_dir(), "the old slot is left where it stands");
11321        assert_eq!(fs::read_to_string(&marker).unwrap(), RETAINED);
11322
11323        // And cleaning the disposable attempt removes its own directory whole
11324        // without reaching the retained slot beside it.
11325        harness.conclude(second.id);
11326        harness.launcher.clean(second.id).await.unwrap();
11327        assert!(!second.runtime_path().exists());
11328        assert!(marker.exists());
11329    }
11330
11331    #[tokio::test]
11332    async fn a_persistent_slot_is_scrubbed_only_after_the_process_is_signalled_and_gone() {
11333        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11334            .with_persistent_workspace(1);
11335        harness.ready().await;
11336        let slot = harness.slot_path(1);
11337        fs::create_dir_all(&slot).unwrap();
11338        let marker = retain_under_work(&slot);
11339        litter_the_slot(&slot);
11340
11341        let id = AttemptId::new_random();
11342        let mut attempt = RunnerAttempt::allocate_in(
11343            id,
11344            harness.policy.id,
11345            &slot,
11346            AttemptWorkspace::persistent_slot(nz(1)),
11347            harness.clock.now(),
11348        );
11349        attempt.jit_received(harness.clock.now()).unwrap();
11350        attempt.started(4242, harness.clock.now()).unwrap();
11351        harness.store.record_attempt(&attempt).unwrap();
11352        harness.clock.advance_secs(11);
11353        harness.processes.set_alive(true);
11354        harness
11355            .github
11356            .observe(GithubRunnerObservation::NotRegistered);
11357
11358        harness.launcher.supervise(&harness.policy).await.unwrap();
11359
11360        // The identity and termination ordering `e3` established is unchanged by
11361        // the workspace kind: the intent is durable before the signal, and the
11362        // slot is scrubbed only once the process is gone.
11363        let actions = harness.processes.actions.lock().unwrap().clone();
11364        let intent = actions
11365            .iter()
11366            .position(|action| *action == "terminate_intent")
11367            .unwrap();
11368        let signal = actions
11369            .iter()
11370            .position(|action| *action == "terminate")
11371            .unwrap();
11372        assert!(intent < signal, "{actions:?}");
11373        assert!(!harness.processes.alive.load(Ordering::SeqCst));
11374
11375        let cleaned = harness.attempt(id);
11376        assert_eq!(cleaned.state(), AttemptState::Cleaned);
11377        assert!(matches!(
11378            cleaned.outcome(),
11379            Some(AttemptOutcome::Failed {
11380                reason: FailureReason::TerminatedAfterRegistrationTimeout
11381            })
11382        ));
11383        assert_eq!(entries_of(&slot), only_the_job_workspace());
11384        assert!(marker.exists());
11385    }
11386
11387    #[test]
11388    fn a_scrub_retains_one_real_work_directory_and_removes_every_other_entry() {
11389        let root = tempfile::tempdir().unwrap();
11390        let slot = root.path().join("s1");
11391        fs::create_dir(&slot).unwrap();
11392        let marker = retain_under_work(&slot);
11393        litter_the_slot(&slot);
11394        fs::write(slot.join("runner-package"), b"verified").unwrap();
11395
11396        scrub_slot_entries(&slot).expect("a slot of ordinary runner state scrubs");
11397        verify_slot_scrubbed(&slot).expect("and proves it afterwards");
11398
11399        assert_eq!(entries_of(&slot), only_the_job_workspace());
11400        assert!(marker.exists());
11401    }
11402
11403    #[test]
11404    fn a_residue_refusal_never_reports_the_under_count_as_the_fact() {
11405        let slot = Path::new("/runners/s1");
11406
11407        // The ordinary case: the listing counted, so the count is the fact and
11408        // the published names qualify it.
11409        let counted = residue_detail(slot, 2, &["`bin`".to_owned()]);
11410        assert!(counted.contains("2 entries other than"), "{counted}");
11411        assert!(counted.contains("including `bin`"), "{counted}");
11412        assert_eq!(
11413            residue_detail(slot, 1, &[]),
11414            format!(
11415                "1 entry other than `{DEFAULT_WORK_FOLDER}` survived cleanup of {}",
11416                slot.display()
11417            )
11418        );
11419
11420        // The race the second pass exists for: the listing saw nothing and the
11421        // filesystem answered otherwise. Saying "0 entries survived" here would
11422        // state the under-count as the fact and contradict the rest of the
11423        // sentence.
11424        let raced = residue_detail(slot, 0, &["`.credentials`".to_owned()]);
11425        assert!(!raced.contains('0'), "{raced}");
11426        assert!(raced.contains("reported nothing but"), "{raced}");
11427        assert!(raced.contains("`.credentials` survived cleanup"), "{raced}");
11428    }
11429
11430    #[test]
11431    fn verification_asks_the_filesystem_rather_than_the_listing_that_missed_an_entry() {
11432        let root = tempfile::tempdir().unwrap();
11433        let slot = root.path().join("s1");
11434        fs::create_dir(&slot).unwrap();
11435        fs::create_dir(slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11436        verify_slot_scrubbed(&slot).expect("only `_work` is a clean slot");
11437
11438        // Runner binaries, registration identity, process identity and this
11439        // agent's lifecycle marks, one at a time, so a scrub that skipped
11440        // exactly one is still caught.
11441        for survivor in ["bin", ".credentials", IDENTITY_FILE, RUNNER_ID_FILE] {
11442            fs::write(slot.join(survivor), b"left behind").unwrap();
11443            let quarantine = verify_slot_scrubbed(&slot).unwrap_err();
11444            assert_eq!(quarantine.refusal, SlotRefusal::Residue);
11445            assert!(
11446                quarantine.detail.contains(&format!("`{survivor}`")),
11447                "{quarantine}"
11448            );
11449            fs::remove_file(slot.join(survivor)).unwrap();
11450        }
11451
11452        // A handoff is named by its published prefix, never by the UUID that
11453        // follows it, and never by the payload it holds.
11454        let handoff = slot.join(format!("{}whatever.tmp", RestrictiveHandoff::NAME_PREFIX));
11455        fs::write(&handoff, JIT.as_bytes()).unwrap();
11456        let quarantine = verify_slot_scrubbed(&slot).unwrap_err();
11457        assert!(
11458            quarantine.detail.contains("an encoded JIT handoff"),
11459            "{quarantine}"
11460        );
11461        assert!(!quarantine.detail.contains(JIT), "{quarantine}");
11462        fs::remove_file(&handoff).unwrap();
11463
11464        // A name a workflow chose is counted and never echoed: a slot root is
11465        // writable by the job, so a file named after a secret would be published
11466        // by any message that repeated the listing.
11467        fs::write(slot.join("ghp_DO_NOT_LEAK"), b"named by the job").unwrap();
11468        let quarantine = verify_slot_scrubbed(&slot).unwrap_err();
11469        assert!(
11470            quarantine.detail.contains("1 entry other than"),
11471            "{quarantine}"
11472        );
11473        assert!(
11474            !quarantine.detail.contains("ghp_DO_NOT_LEAK"),
11475            "{quarantine}"
11476        );
11477    }
11478
11479    #[test]
11480    fn a_slot_is_derived_from_the_journal_and_refused_when_it_disagrees() {
11481        let root = tempfile::tempdir().unwrap();
11482        let configured =
11483            LocalAbsolutePath::new(root.path().to_str().unwrap()).expect("a local absolute root");
11484        let slot = configured.as_path().join("s1");
11485        fs::create_dir(&slot).unwrap();
11486
11487        verify_journalled_slot(&slot, nz(1), Some(&configured))
11488            .expect("the journalled slot agrees");
11489        verify_journalled_slot(&slot, nz(1), None)
11490            .expect("and a policy that is gone removes a check, not the ability to clean");
11491
11492        // The journalled slot number is what names the directory. `s1` recorded
11493        // as slot two is corrupt state, not a slot to clean.
11494        assert_eq!(
11495            verify_journalled_slot(&slot, nz(2), None)
11496                .unwrap_err()
11497                .refusal,
11498            SlotRefusal::NotTheJournalledSlot
11499        );
11500        for stray in ["s1/nested", "not-a-slot", "s01"] {
11501            let path = configured.as_path().join(stray);
11502            assert_eq!(
11503                verify_journalled_slot(&path, nz(1), None)
11504                    .unwrap_err()
11505                    .refusal,
11506                SlotRefusal::NotTheJournalledSlot,
11507                "{}",
11508                path.display()
11509            );
11510        }
11511
11512        // A surviving policy that names a different root does not get to have
11513        // its disagreement resolved by deleting something.
11514        let elsewhere = tempfile::tempdir().unwrap();
11515        let other =
11516            LocalAbsolutePath::new(elsewhere.path().to_str().unwrap()).expect("a second root");
11517        assert_eq!(
11518            verify_journalled_slot(&slot, nz(1), Some(&other))
11519                .unwrap_err()
11520                .refusal,
11521            SlotRefusal::PolicyRootDisagrees
11522        );
11523    }
11524
11525    #[cfg(unix)]
11526    #[test]
11527    fn a_substituted_work_directory_quarantines_the_slot_and_deletes_nothing_outside_it() {
11528        // Windows needs a privilege to create a junction or a symlink, so the
11529        // substitution is made here; the rule is platform-independent because it
11530        // is `symlink_metadata`'s answer plus the reparse attribute.
11531        let root = tempfile::tempdir().unwrap();
11532        let outside = root.path().join("operator-data");
11533        fs::create_dir(&outside).unwrap();
11534        let sentinel = outside.join("do-not-delete.txt");
11535        fs::write(
11536            &sentinel,
11537            b"an operator's data, outside every approved root",
11538        )
11539        .unwrap();
11540
11541        let slot = root.path().join("s1");
11542        fs::create_dir(&slot).unwrap();
11543        fs::create_dir(slot.join("bin")).unwrap();
11544        std::os::unix::fs::symlink(&outside, slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11545
11546        let quarantine = scrub_slot_entries(&slot).unwrap_err();
11547        assert_eq!(quarantine.refusal, SlotRefusal::WorkNotADirectory);
11548        assert!(
11549            sentinel.exists(),
11550            "the deletion followed the link out of the slot"
11551        );
11552        assert!(outside.is_dir());
11553        assert!(
11554            slot.join(DEFAULT_WORK_FOLDER).symlink_metadata().is_ok(),
11555            "the substituted link is left for the operator, never unlinked as if it were ours"
11556        );
11557
11558        // A `_work` that is a plain file is the same refusal for the same
11559        // reason: it is not a job workspace, and this is not the code that
11560        // decides what to do about it.
11561        let file_work = root.path().join("s2");
11562        fs::create_dir(&file_work).unwrap();
11563        fs::write(file_work.join(DEFAULT_WORK_FOLDER), b"not a directory").unwrap();
11564        assert_eq!(
11565            scrub_slot_entries(&file_work).unwrap_err().refusal,
11566            SlotRefusal::WorkNotADirectory
11567        );
11568    }
11569
11570    #[cfg(unix)]
11571    #[test]
11572    fn a_slot_replaced_by_a_link_out_of_its_root_is_refused_before_anything_is_read() {
11573        let root = tempfile::tempdir().unwrap();
11574        let outside = root.path().join("operator-data");
11575        fs::create_dir(&outside).unwrap();
11576        let sentinel = outside.join("do-not-delete.txt");
11577        fs::write(
11578            &sentinel,
11579            b"an operator's data, outside every approved root",
11580        )
11581        .unwrap();
11582
11583        // The lexical half of containment passes -- the name is right and the
11584        // parent is right -- and canonical resolution is what catches it.
11585        let inside = root.path().join("inside");
11586        fs::create_dir(&inside).unwrap();
11587        let slot = inside.join("s1");
11588        std::os::unix::fs::symlink(&outside, &slot).unwrap();
11589
11590        assert_eq!(
11591            verify_journalled_slot(&slot, nz(1), None)
11592                .unwrap_err()
11593                .refusal,
11594            SlotRefusal::Containment
11595        );
11596        assert!(sentinel.exists());
11597        assert!(
11598            slot.symlink_metadata().is_ok(),
11599            "the link is left for the operator rather than removed as if it were ours"
11600        );
11601    }
11602
11603    /// The Windows half of the case above.
11604    ///
11605    /// Containment was proven only on Unix, and a symbolic link is the wrong
11606    /// instrument to prove it with on Windows: creating one needs a privilege an
11607    /// ordinary workflow does not have, so it is not the substitution an
11608    /// attacker would reach for. A **junction** needs none, which makes it both
11609    /// the realistic attack and the one this repository must refuse -- and
11610    /// `04-security-recovery.md` names it in the same breath as the symlink for
11611    /// exactly that reason.
11612    #[cfg(windows)]
11613    #[test]
11614    fn a_slot_root_replaced_by_a_junction_is_refused_before_anything_is_read() {
11615        let root = tempfile::tempdir().unwrap();
11616        let outside = root.path().join("operator-data");
11617        fs::create_dir(&outside).unwrap();
11618        let sentinel = outside.join("do-not-delete.txt");
11619        fs::write(
11620            &sentinel,
11621            b"an operator's data, outside every approved root",
11622        )
11623        .unwrap();
11624
11625        // The lexical half of containment passes -- `s1` under the root the
11626        // journal names -- and canonical resolution is what catches it.
11627        let inside = root.path().join("inside");
11628        fs::create_dir(&inside).unwrap();
11629        let slot = inside.join("s1");
11630        let Some(()) = plant_junction(&slot, &outside) else {
11631            eprintln!("skipped: this machine would not create a directory junction");
11632            return;
11633        };
11634
11635        assert_eq!(
11636            verify_journalled_slot(&slot, nz(1), None)
11637                .unwrap_err()
11638                .refusal,
11639            SlotRefusal::Containment
11640        );
11641        assert!(
11642            sentinel.exists(),
11643            "the refusal resolved the junction and reached the operator's data"
11644        );
11645        assert!(
11646            slot.symlink_metadata().is_ok(),
11647            "the junction is left for the operator rather than removed as if it were ours"
11648        );
11649    }
11650
11651    /// Plant a directory junction at `link` pointing at `target`.
11652    ///
11653    /// A junction is the Windows substitution this has to refuse, and unlike a
11654    /// symbolic link it needs no privilege — which is exactly why it is the one
11655    /// an unprivileged workflow would reach for. `mklink` is a `cmd` builtin, so
11656    /// there is no binary to find and nothing to install; `None` means this
11657    /// machine would not make one and the caller says so rather than asserting
11658    /// nothing.
11659    #[cfg(windows)]
11660    fn plant_junction(link: &Path, target: &Path) -> Option<()> {
11661        let made = std::process::Command::new("cmd")
11662            .arg("/C")
11663            .arg("mklink")
11664            .arg("/J")
11665            .arg(link)
11666            .arg(target)
11667            .output()
11668            .ok()?;
11669        (made.status.success() && link.symlink_metadata().is_ok()).then_some(())
11670    }
11671
11672    /// The dependency caches live beside attempts, under `<runner root>/_cache`,
11673    /// and jobs link into them (pnpm hard-links from its store; a workflow may
11674    /// symlink or junction a cache into its workspace). Removing an ephemeral
11675    /// attempt must take the links and never what they point at.
11676    #[test]
11677    fn removing_an_attempt_never_reaches_into_the_dependency_caches_it_linked() {
11678        let root = tempfile::tempdir().unwrap();
11679        let cache = root.path().join("_cache").join("o").join("r").join("npm");
11680        fs::create_dir_all(&cache).unwrap();
11681        let cached = cache.join("package.tgz");
11682        fs::write(&cached, b"kept between jobs").unwrap();
11683        let attempt = root.path().join("0123456789abcdef");
11684        let work = attempt.join("_work").join("repo");
11685        fs::create_dir_all(&work).unwrap();
11686        fs::hard_link(&cached, work.join("hard-linked.tgz")).unwrap();
11687        #[cfg(unix)]
11688        std::os::unix::fs::symlink(&cache, work.join("linked-cache")).unwrap();
11689        #[cfg(windows)]
11690        if plant_junction(&work.join("linked-cache"), &cache).is_none() {
11691            eprintln!("this machine would not create a junction; the hard link is still checked");
11692        }
11693
11694        remove_runtime_tree(&attempt).unwrap();
11695        assert!(!attempt.exists());
11696        assert_eq!(fs::read(&cached).unwrap(), b"kept between jobs");
11697    }
11698
11699    #[cfg(windows)]
11700    #[test]
11701    fn a_work_directory_replaced_by_a_junction_fails_closed_and_deletes_nothing_beyond_it() {
11702        let root = tempfile::tempdir().unwrap();
11703        let outside = root.path().join("operator-data");
11704        fs::create_dir(&outside).unwrap();
11705        let sentinel = outside.join("do-not-delete.txt");
11706        fs::write(
11707            &sentinel,
11708            b"an operator's data, outside every approved root",
11709        )
11710        .unwrap();
11711
11712        let slot = root.path().join("s1");
11713        fs::create_dir(&slot).unwrap();
11714        fs::create_dir(slot.join("bin")).unwrap();
11715        let Some(()) = plant_junction(&slot.join(DEFAULT_WORK_FOLDER), &outside) else {
11716            eprintln!("skipped: this machine would not create a directory junction");
11717            return;
11718        };
11719
11720        // The reparse point is what `is_link_like` answers on, so a junction is
11721        // refused for the same reason a symbolic link is and neither is
11722        // descended into.
11723        let work = fs::symlink_metadata(slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11724        assert!(is_link_like(&work), "a junction is a reparse point");
11725        let quarantine = scrub_slot_entries(&slot).unwrap_err();
11726        assert_eq!(quarantine.refusal, SlotRefusal::WorkNotADirectory);
11727        assert!(
11728            sentinel.exists(),
11729            "the deletion followed the junction out of the slot"
11730        );
11731        assert!(outside.is_dir());
11732
11733        // A junction standing where an ordinary entry was is unlinked rather
11734        // than followed, so the removal still cannot reach through it.
11735        let elsewhere = root.path().join("s2");
11736        fs::create_dir(&elsewhere).unwrap();
11737        fs::create_dir(elsewhere.join(DEFAULT_WORK_FOLDER)).unwrap();
11738        if plant_junction(&elsewhere.join("externals"), &outside).is_some() {
11739            scrub_slot_entries(&elsewhere).expect("an ordinary entry is removed, junction or not");
11740            verify_slot_scrubbed(&elsewhere).expect("and the slot verifies");
11741            assert!(sentinel.exists(), "the junction was followed, not unlinked");
11742            assert_eq!(entries_of(&elsewhere), only_the_job_workspace());
11743        }
11744    }
11745
11746    #[cfg(unix)]
11747    #[tokio::test]
11748    async fn a_substituted_work_directory_leaves_the_attempt_uncleaned_and_still_leased() {
11749        let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11750            .with_persistent_workspace(2);
11751        harness.ready().await;
11752        let first = harness.launch().await;
11753        let slot = harness.slot_path(1);
11754        harness.conclude(first.id);
11755
11756        let outside = harness._root.path().join("operator-data");
11757        fs::create_dir_all(&outside).unwrap();
11758        let sentinel = outside.join("do-not-delete.txt");
11759        fs::write(&sentinel, b"outside every approved root").unwrap();
11760        std::os::unix::fs::symlink(&outside, slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11761
11762        harness
11763            .launcher
11764            .clean(first.id)
11765            .await
11766            .expect_err("a slot whose `_work` was substituted is quarantined");
11767        assert!(sentinel.exists());
11768
11769        let held = harness.attempt(first.id);
11770        assert_eq!(held.state(), AttemptState::Failed);
11771        assert!(held.holds_slot_lease());
11772
11773        // The quarantined slot is not silently chosen again.
11774        let second = harness.launch().await;
11775        assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
11776    }
11777
11778    #[test]
11779    fn a_slot_that_is_a_file_is_refused_and_a_slot_that_is_gone_is_not() {
11780        let root = tempfile::tempdir().unwrap();
11781        let occupied = root.path().join("s1");
11782        fs::write(&occupied, b"an operator's file").unwrap();
11783        // The journal check passes -- it is the right name under the right root
11784        // -- and the shape check is what refuses.
11785        verify_journalled_slot(&occupied, nz(1), None).expect("the path is the journalled slot");
11786        assert_eq!(
11787            slot_is_present(&occupied).unwrap_err().refusal,
11788            SlotRefusal::SlotNotADirectory
11789        );
11790        assert_eq!(fs::read_to_string(&occupied).unwrap(), "an operator's file");
11791
11792        // A directory that is simply not there leaves nothing to remove and
11793        // nothing to prove absent, so it is not a refusal.
11794        assert!(!slot_is_present(&root.path().join("s2")).unwrap());
11795        let present = root.path().join("s3");
11796        fs::create_dir(&present).unwrap();
11797        assert!(slot_is_present(&present).unwrap());
11798    }
11799
11800    #[test]
11801    fn cleanup_dispatches_on_the_journalled_kind_and_not_on_what_the_directory_holds() {
11802        let root = tempfile::tempdir().unwrap();
11803
11804        // A disposable directory that happens to contain a `_work` still goes
11805        // whole: the workspace kind is immutable so that the shape of a
11806        // directory a workflow can write to cannot choose its own algorithm.
11807        let disposable = root.path().join("abcdef012345");
11808        fs::create_dir_all(disposable.join(DEFAULT_WORK_FOLDER).join("repo")).unwrap();
11809        let ephemeral = RunnerAttempt::allocate(
11810            AttemptId::new_random(),
11811            fixtures::POLICY_ID,
11812            &disposable,
11813            fixtures::created_at(),
11814        );
11815        remove_materialized_package(&ephemeral).unwrap();
11816        assert!(!disposable.exists());
11817
11818        // And a slot keeps its `_work` with the same contents beneath it.
11819        let slot = root.path().join("s1");
11820        fs::create_dir_all(slot.join(DEFAULT_WORK_FOLDER).join("repo")).unwrap();
11821        fs::create_dir_all(slot.join("bin")).unwrap();
11822        let persistent = RunnerAttempt::allocate_in(
11823            AttemptId::new_random(),
11824            fixtures::POLICY_ID,
11825            &slot,
11826            AttemptWorkspace::persistent_slot(nz(1)),
11827            fixtures::created_at(),
11828        );
11829        remove_materialized_package(&persistent).unwrap();
11830        assert_eq!(entries_of(&slot), only_the_job_workspace());
11831        assert!(slot.join(DEFAULT_WORK_FOLDER).join("repo").is_dir());
11832    }
11833
11834    #[test]
11835    fn every_slot_refusal_names_a_distinct_event_class_and_keeps_the_lease() {
11836        let refusals = [
11837            SlotRefusal::NotTheJournalledSlot,
11838            SlotRefusal::PolicyRootDisagrees,
11839            SlotRefusal::Containment,
11840            SlotRefusal::SlotNotADirectory,
11841            SlotRefusal::Enumeration,
11842            SlotRefusal::WorkNotADirectory,
11843            SlotRefusal::Deletion,
11844            SlotRefusal::Residue,
11845        ];
11846        let classes: BTreeSet<&str> = refusals.iter().map(|refusal| refusal.class()).collect();
11847        assert_eq!(
11848            classes.len(),
11849            refusals.len(),
11850            "an event class shared by two refusals tells an operator less than it appears to"
11851        );
11852        for refusal in refusals {
11853            // The event field is a closed vocabulary, so it has to look like
11854            // one: `d1`'s sink allows the name verbatim.
11855            assert!(
11856                refusal
11857                    .class()
11858                    .chars()
11859                    .all(|c| c.is_ascii_lowercase() || c == '_'),
11860                "{}",
11861                refusal.class()
11862            );
11863            assert!(
11864                refusal.remediation().contains("slot lease"),
11865                "every refusal has to say the lease is still held: {}",
11866                refusal.class()
11867            );
11868        }
11869    }
11870
11871    #[test]
11872    fn copy_package_tree_copies_files_and_preserves_paths_with_spaces() {
11873        let root = tempfile::tempdir().unwrap();
11874        let source = root.path().join("source with spaces");
11875        let dest = root.path().join("dest with spaces");
11876
11877        fs::create_dir_all(&source).unwrap();
11878        fs::write(source.join("file1.txt"), b"hello").unwrap();
11879
11880        let nested = source.join("nested dir");
11881        fs::create_dir_all(&nested).unwrap();
11882        fs::write(nested.join("file2.txt"), b"world").unwrap();
11883
11884        // This is not a top-level `_work`, so it should be allowed
11885        let nested_work = nested.join(DEFAULT_WORK_FOLDER);
11886        fs::create_dir_all(&nested_work).unwrap();
11887        fs::write(nested_work.join("allowed.txt"), b"allowed").unwrap();
11888
11889        copy_package_tree(&source, &dest, &TrustedTrees::default()).unwrap();
11890
11891        assert_eq!(fs::read_to_string(dest.join("file1.txt")).unwrap(), "hello");
11892        assert_eq!(
11893            fs::read_to_string(dest.join("nested dir").join("file2.txt")).unwrap(),
11894            "world"
11895        );
11896        assert_eq!(
11897            fs::read_to_string(
11898                dest.join("nested dir")
11899                    .join(DEFAULT_WORK_FOLDER)
11900                    .join("allowed.txt")
11901            )
11902            .unwrap(),
11903            "allowed"
11904        );
11905    }
11906
11907    #[test]
11908    fn every_field_name_the_attempt_sink_emits_is_allowed() {
11909        use runner_manager_platform::logging::is_field_allowed;
11910
11911        // Kept beside the sink rather than derived from it, as in `reconcile`.
11912        for field in [
11913            "event",
11914            "attempt_id",
11915            "reason",
11916            "retry_in_ms",
11917            "duration_ms",
11918            "order_wait_ms",
11919            "lock_wait_ms",
11920            "turn_wait_ms",
11921            "materialize_ms",
11922            "package_wait_ms",
11923            "package_install_ms",
11924            "package_verify_ms",
11925            "package_copy_ms",
11926            "package_downloaded",
11927            "prepare_ms",
11928            "register_ms",
11929            "spawn_ms",
11930        ] {
11931            assert!(
11932                is_field_allowed(field),
11933                "`{field}` would be logged as `[redacted]`"
11934            );
11935        }
11936    }
11937
11938    #[cfg(unix)]
11939    fn package_entry(root: &Path, version: &str, manifest: &str) -> PathBuf {
11940        let entry = root.join("packages").join(version);
11941        fs::create_dir_all(entry.join("bin")).unwrap();
11942        fs::write(entry.join(crate::package::MANIFEST_FILE), manifest).unwrap();
11943        fs::write(entry.join("bin").join("Runner.Listener"), version).unwrap();
11944        entry
11945    }
11946
11947    /// The seed's binary's inode: a rebuilt seed holds a new file.
11948    #[cfg(unix)]
11949    fn listener_inode(seed: &Path) -> u64 {
11950        use std::os::unix::fs::MetadataExt as _;
11951        fs::metadata(seed.join("bin").join("Runner.Listener"))
11952            .unwrap()
11953            .ino()
11954    }
11955
11956    #[cfg(unix)]
11957    #[test]
11958    fn a_package_seed_is_built_once_and_reused_while_untouched() {
11959        let root = tempfile::tempdir().unwrap();
11960        let source = package_entry(root.path(), "2.336.0", "{\"digest\":\"a\"}");
11961        let seeds = root.path().join("runners").join(PACKAGE_SEED_DIR);
11962        let trusted = TrustedTrees::default();
11963
11964        let seed = refresh_package_seed(&source, &seeds, &trusted).unwrap();
11965        assert_eq!(seed, seeds.join("2.336.0"));
11966        assert_eq!(
11967            fs::read_to_string(seed.join("bin").join("Runner.Listener")).unwrap(),
11968            "2.336.0"
11969        );
11970
11971        let built = listener_inode(&seed);
11972        assert_eq!(
11973            refresh_package_seed(&source, &seeds, &trusted).unwrap(),
11974            seed
11975        );
11976        assert_eq!(listener_inode(&seed), built, "an untouched seed is reused");
11977    }
11978
11979    #[cfg(unix)]
11980    #[test]
11981    fn a_tampered_or_unrecorded_seed_is_rebuilt() {
11982        let root = tempfile::tempdir().unwrap();
11983        let source = package_entry(root.path(), "2.336.0", "{\"digest\":\"a\"}");
11984        let seeds = root.path().join("runners").join(PACKAGE_SEED_DIR);
11985        let trusted = TrustedTrees::default();
11986        let seed = refresh_package_seed(&source, &seeds, &trusted).unwrap();
11987
11988        // A job rewrites the binary in place and leaves the manifest alone.
11989        fs::write(seed.join("bin").join("Runner.Listener"), b"planted").unwrap();
11990        refresh_package_seed(&source, &seeds, &trusted).unwrap();
11991        assert_eq!(
11992            fs::read_to_string(seed.join("bin").join("Runner.Listener")).unwrap(),
11993            "2.336.0",
11994            "a seed whose print moved is rebuilt from the cache entry"
11995        );
11996
11997        // A restarted daemon has no print of a seed it did not build.
11998        // Keep the old inode allocated until the replacement is complete;
11999        // otherwise the filesystem may immediately reuse its number.
12000        let old_listener = fs::File::open(seed.join("bin").join("Runner.Listener")).unwrap();
12001        let built = listener_inode(&seed);
12002        refresh_package_seed(&source, &seeds, &TrustedTrees::default()).unwrap();
12003        assert_ne!(
12004            listener_inode(&seed),
12005            built,
12006            "an unrecorded seed is rebuilt"
12007        );
12008        drop(old_listener);
12009    }
12010
12011    #[cfg(unix)]
12012    #[test]
12013    fn a_package_seed_is_rebuilt_for_a_new_entry_and_drops_every_other() {
12014        let root = tempfile::tempdir().unwrap();
12015        let seeds = root.path().join("runners").join(PACKAGE_SEED_DIR);
12016        let trusted = TrustedTrees::default();
12017        let old = package_entry(root.path(), "2.335.0", "{\"digest\":\"old\"}");
12018        refresh_package_seed(&old, &seeds, &trusted).unwrap();
12019        fs::create_dir_all(seeds.join(".staging-interrupted")).unwrap();
12020        fs::write(seeds.join("stray"), b"").unwrap();
12021
12022        let new = package_entry(root.path(), "2.336.0", "{\"digest\":\"new\"}");
12023        let seed = refresh_package_seed(&new, &seeds, &trusted).unwrap();
12024
12025        let mut left: Vec<_> = fs::read_dir(&seeds)
12026            .unwrap()
12027            .map(|entry| entry.unwrap().file_name().into_string().unwrap())
12028            .collect();
12029        left.sort();
12030        assert_eq!(left, ["2.336.0"]);
12031
12032        // A reinstalled entry of the same version has a different manifest.
12033        fs::write(
12034            new.join(crate::package::MANIFEST_FILE),
12035            "{\"digest\":\"again\"}",
12036        )
12037        .unwrap();
12038        refresh_package_seed(&new, &seeds, &trusted).unwrap();
12039        assert_eq!(
12040            fs::read_to_string(seed.join(crate::package::MANIFEST_FILE)).unwrap(),
12041            "{\"digest\":\"again\"}",
12042            "a stale seed is rebuilt from the changed entry"
12043        );
12044    }
12045
12046    #[cfg(unix)]
12047    #[test]
12048    fn a_package_on_the_runtime_volume_is_its_own_seed() {
12049        let root = tempfile::tempdir().unwrap();
12050        let source = package_entry(root.path(), "2.336.0", "{}");
12051        let runtime = root.path().join("runners").join("attempt");
12052        fs::create_dir_all(&runtime).unwrap();
12053
12054        let trusted = TrustedTrees::default();
12055        assert_eq!(package_seed(&source, &runtime, &trusted).unwrap(), source);
12056        copy_package_tree(&source, &runtime, &trusted).unwrap();
12057        assert!(runtime.join("bin").join("Runner.Listener").exists());
12058        assert!(!root.path().join("runners").join(PACKAGE_SEED_DIR).exists());
12059    }
12060
12061    #[test]
12062    fn copy_package_tree_refuses_top_level_work_folder() {
12063        let root = tempfile::tempdir().unwrap();
12064        let source = root.path().join("source");
12065        let dest = root.path().join("dest");
12066
12067        fs::create_dir_all(&source).unwrap();
12068        fs::write(source.join("file1.txt"), b"hello").unwrap();
12069
12070        // Top-level `_work` should be refused
12071        let top_work = source.join(DEFAULT_WORK_FOLDER);
12072        fs::create_dir_all(&top_work).unwrap();
12073
12074        let err = copy_package_tree(&source, &dest, &TrustedTrees::default()).unwrap_err();
12075        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
12076    }
12077}