1#[cfg(test)]
12use std::collections::VecDeque;
13use std::collections::{BTreeMap, BTreeSet, HashMap};
14use std::ffi::OsStr;
15use std::fmt;
16use std::fs;
17use std::io::Write;
18use std::num::NonZeroU16;
19use std::path::{Path, PathBuf};
20use std::sync::{Arc, Mutex};
21use std::time::Duration;
22
23use async_trait::async_trait;
24use runner_manager_domain::attempt::{
25 AttemptOutcome, AttemptState, FailureReason, GithubRunnerObservation, IsolationProviderFailure,
26 RecoveryDecision, RecoveryObservation, RecoveryTimeouts, RunnerAttempt, authorize,
27 recovery_decision,
28};
29use runner_manager_domain::execution::{AttemptExecution, Backend, ImageReference};
30use runner_manager_domain::model::{AttemptId, Clock, HostId, PolicyId, ScaleTarget};
31use runner_manager_domain::path::LocalAbsolutePath;
32use runner_manager_domain::policy::ScalePolicy;
33use runner_manager_domain::store::{Store, StoreError};
34use runner_manager_domain::workspace::{AttemptWorkspace, WorkspacePolicy};
35use runner_manager_github::jit::{
36 DEFAULT_WORK_FOLDER, EncodedJitConfig, JitError, JitGateway, JitRegistration, JitRunnerRequest,
37};
38use runner_manager_github::rest::{CancelToken, InventoryGateway};
39use runner_manager_platform::process::{
40 Adoption, ChildProcess, ProcessIdentity, RestrictiveHandoff, SpawnSpec, Termination,
41};
42use runner_manager_platform::runner_account::{RunnerAccount, RunnerPool};
43use runner_manager_platform::runner_env::{self, RunnerEnv, RunnerPlatform};
44use runner_manager_platform::runner_root::{
45 self, RootOwner, RootPreflight, RunnerRootError, default_runner_root,
46};
47use secrecy::SecretString;
48
49use crate::dependency_caches::DependencyCaches;
50use crate::oci::OciProcesses;
51use crate::package::{
52 InstalledPackage, LastCheck, PackageCache, PackageContents, PackageError, RunnerVersion,
53};
54use crate::reconcile::{
55 AllocationGuard, CleanBackoff, EventSink, LaunchFailure, LaunchRequest, LifecycleEvent,
56 OutcomeKind, ReplacementIntent, ReservationWaits, RunnerLauncher, failure_reason_kind,
57};
58
59mod windows_hyperv;
60pub use windows_hyperv::{WindowsHyperVContainers, WindowsHyperVHostState};
61
62const IDENTITY_FILE: &str = ".runner-process.json";
63const FALLBACK_IDENTITY_FILE: &str = ".runner-process.recovery.json";
64const UNRESOLVED_PROCESS_FILE: &str = ".runner-process.unresolved";
65const RUNNER_ID_FILE: &str = ".github-runner-id";
66const TERMINATE_INTENT_FILE: &str = ".terminate-registration-timeout";
67const MAX_POST_SPAWN_STOP_ATTEMPTS: usize = 3;
68
69const SENSITIVE_SLOT_ENTRIES: &[&str] = &[
82 "bin",
84 "externals",
85 "run.sh",
86 "run.cmd",
87 "config.sh",
88 "config.cmd",
89 ".runner",
92 ".credentials",
93 ".credentials_rsaparams",
94 ".env",
95 ".path",
96 "_diag",
97 IDENTITY_FILE,
99 FALLBACK_IDENTITY_FILE,
100 UNRESOLVED_PROCESS_FILE,
101 RUNNER_ID_FILE,
102 TERMINATE_INTENT_FILE,
103];
104#[cfg(test)]
105const TEST_LISTENER_READY: &str = ".test-listener-ready";
106
107fn runner_listener_spec(
119 program: PathBuf,
120 runtime: &Path,
121 caches: Vec<(&'static str, std::ffi::OsString)>,
122 host_env: &RunnerEnv,
123) -> SpawnSpec {
124 let tmp = runtime.join("tmp");
125 let _ = std::fs::create_dir_all(&tmp);
126 let platform = RunnerPlatform::current();
127 let defaults = runner_env::platform_defaults(
128 platform,
129 runtime,
130 &runner_env::Inherited::current(),
131 Path::exists,
132 );
133 for dir in runner_env::directories_to_create(platform, &defaults) {
137 let _ = std::fs::create_dir_all(dir);
138 }
139 let mut spec = SpawnSpec::new(program)
140 .arg("run")
141 .working_dir(runtime)
142 .expect_normal_scheduling();
143 for (name, value) in runner_env::runner_environment(defaults, caches, host_env) {
144 spec = spec.env(name, value);
145 }
146 spec.env("TMPDIR", &tmp).env("TEMP", &tmp).env("TMP", &tmp)
147}
148
149const DOCKER_SOCKET_DIR: &str = ".docker-proxy";
153
154fn hand_over(
162 account: RunnerAccount,
163 pool: &RunnerPool,
164 runtime: &Path,
165 spec: &SpawnSpec,
166 cache_names: &[&str],
167) -> std::io::Result<()> {
168 hand_over_attempt(account, runtime)?;
169 for name in cache_names {
170 let Some(value) = spec.configured_env(name) else {
171 continue;
172 };
173 let path = Path::new(value);
174 if !path.is_absolute() {
175 continue;
176 }
177 match std::fs::symlink_metadata(path) {
178 Ok(metadata) if metadata.is_dir() && !pool.is_shared(path) => {
179 pool.share_tree(path)?;
180 }
181 Ok(_) => {}
182 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
183 Err(error) => return Err(error),
184 }
185 }
186 Ok(())
187}
188
189fn hand_over_attempt(account: RunnerAccount, runtime: &Path) -> std::io::Result<()> {
192 account.give_tree(runtime)?;
193 #[cfg(unix)]
194 {
195 use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _};
196 fs::set_permissions(runtime, fs::Permissions::from_mode(0o700))?;
203 if let Some(root) = runtime.parent() {
204 let metadata = fs::symlink_metadata(root)?;
205 if metadata.is_dir() && metadata.uid() == 0 {
206 if metadata.gid() != account.gid {
207 std::os::unix::fs::lchown(root, None, Some(account.gid))?;
208 }
209 if metadata.mode() & 0o7777 != 0o750 {
210 fs::set_permissions(root, fs::Permissions::from_mode(0o750))?;
211 }
212 }
213 }
214 }
215 Ok(())
216}
217
218fn owner_in_pool(pool: &RunnerPool, path: &Path) -> Option<RunnerAccount> {
220 #[cfg(unix)]
221 {
222 use std::os::unix::fs::MetadataExt as _;
223 let metadata = fs::symlink_metadata(path).ok()?;
224 pool.account(metadata.uid()).filter(|_| metadata.is_dir())
225 }
226 #[cfg(not(unix))]
227 {
228 let _ = (pool, path);
229 None
230 }
231}
232
233#[derive(Debug)]
235enum RecordedProcess {
236 None,
238 Runner(ProcessIdentity),
240 Foreign,
243}
244
245impl RecordedProcess {
246 fn runner(self) -> Option<ProcessIdentity> {
247 match self {
248 Self::Runner(identity) => Some(identity),
249 Self::None | Self::Foreign => None,
250 }
251 }
252}
253
254#[derive(Debug, Clone, Default)]
256enum RunnerIdentity {
257 #[default]
259 Daemon,
260 Pool(RunnerPool),
262 Missing(String),
265}
266
267impl RunnerIdentity {
268 fn pool(&self) -> Result<Option<&RunnerPool>, FailureReason> {
271 match self {
272 Self::Daemon => Ok(None),
273 Self::Pool(pool) => Ok(Some(pool)),
274 Self::Missing(why) => Err(FailureReason::Other(why.clone())),
275 }
276 }
277}
278
279#[derive(Debug, Clone, Copy, PartialEq, Eq)]
281pub struct RetryPolicy {
282 pub max_attempts: u32,
283 pub initial: Duration,
284 pub maximum: Duration,
285}
286
287impl RetryPolicy {
288 #[must_use]
289 pub const fn bounded(max_attempts: u32, initial: Duration, maximum: Duration) -> Self {
290 Self {
291 max_attempts,
292 initial,
293 maximum,
294 }
295 }
296
297 pub(crate) fn delay(self, failure_index: u32) -> Duration {
298 let shift = failure_index.saturating_sub(1).min(31);
299 self.initial
300 .saturating_mul(1_u32 << shift)
301 .min(self.maximum)
302 }
303}
304
305#[derive(Debug, Clone, PartialEq, Eq)]
308pub enum AttemptEvent {
309 State {
310 attempt: AttemptId,
311 state: AttemptState,
312 },
313 Retry {
314 attempt: AttemptId,
315 operation: &'static str,
316 delay: Duration,
317 },
318 Launched {
322 attempt: AttemptId,
323 started: bool,
324 timings: LaunchTimings,
325 },
326 Adopted {
327 attempt: AttemptId,
328 },
329 RemoteIdentityRecovered {
330 attempt: AttemptId,
331 runner_id: u64,
332 },
333 TerminateIntent {
334 attempt: AttemptId,
335 },
336 Terminated {
337 attempt: AttemptId,
338 },
339 Deregistered {
343 attempt: AttemptId,
344 runner_id: u64,
345 },
346 Concluded {
347 attempt: AttemptId,
348 outcome: OutcomeKind,
349 },
350 Cleaned {
351 attempt: AttemptId,
352 outcome: OutcomeKind,
353 },
354 QuarantinedOwnedResource {
355 attempt: AttemptId,
356 },
357}
358
359pub trait AttemptEventSink: fmt::Debug + Send + Sync {
360 fn emit(&self, event: AttemptEvent);
361}
362
363#[derive(Debug, Default)]
364pub struct AttemptEventLog(Mutex<Vec<AttemptEvent>>);
365
366impl AttemptEventLog {
367 #[must_use]
368 pub fn events(&self) -> Vec<AttemptEvent> {
369 self.0
370 .lock()
371 .map(|events| events.clone())
372 .unwrap_or_default()
373 }
374}
375
376impl AttemptEventSink for AttemptEventLog {
377 fn emit(&self, event: AttemptEvent) {
378 if let Ok(mut events) = self.0.lock() {
379 events.push(event);
380 }
381 }
382}
383
384#[derive(Debug, Clone, Copy, Default)]
385pub struct NoAttemptEvents;
386
387impl AttemptEventSink for NoAttemptEvents {
388 fn emit(&self, _event: AttemptEvent) {}
389}
390
391#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
393pub struct LaunchTimings {
394 pub order_wait: Duration,
396 pub lock_wait: Duration,
399 pub turn_wait: Duration,
401 pub materialize: Duration,
402 pub package: PackageTimings,
404 pub prepare: Duration,
407 pub register: Duration,
408 pub spawn: Duration,
409}
410
411impl LaunchTimings {
412 #[must_use]
413 pub fn total(&self) -> Duration {
414 self.order_wait
415 + self.lock_wait
416 + self.turn_wait
417 + self.materialize
418 + self.prepare
419 + self.register
420 + self.spawn
421 }
422}
423
424#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
426pub struct PackageTimings {
427 pub wait: Duration,
430 pub install: Duration,
432 pub verify: Duration,
434 pub copy: Duration,
436 pub downloaded: bool,
438}
439
440#[derive(Debug, Clone, Copy)]
441enum LaunchStep {
442 Turn,
443 Materialize,
444 Prepare,
445 Register,
446 Spawn,
447}
448
449#[derive(Debug)]
453struct LaunchStopwatch {
454 timings: LaunchTimings,
455 running: Option<(LaunchStep, std::time::Instant)>,
456}
457
458impl LaunchStopwatch {
459 fn new(waits: ReservationWaits) -> Self {
460 Self {
461 timings: LaunchTimings {
462 order_wait: waits.order,
463 lock_wait: waits.lock,
464 ..LaunchTimings::default()
465 },
466 running: None,
467 }
468 }
469
470 fn begin(&mut self, step: LaunchStep) {
471 self.finish();
472 self.running = Some((step, std::time::Instant::now()));
473 }
474
475 fn finish(&mut self) {
476 if let Some((step, since)) = self.running.take() {
477 let slot = match step {
478 LaunchStep::Turn => &mut self.timings.turn_wait,
479 LaunchStep::Materialize => &mut self.timings.materialize,
480 LaunchStep::Prepare => &mut self.timings.prepare,
481 LaunchStep::Register => &mut self.timings.register,
482 LaunchStep::Spawn => &mut self.timings.spawn,
483 };
484 *slot = since.elapsed();
485 }
486 }
487}
488
489const SLOW_LAUNCH: Duration = Duration::from_secs(60);
494
495#[derive(Debug, Clone, Copy, Default)]
502pub struct TracingAttemptEvents;
503
504impl AttemptEventSink for TracingAttemptEvents {
505 fn emit(&self, event: AttemptEvent) {
506 match event {
507 AttemptEvent::Retry {
508 attempt,
509 operation,
510 delay,
511 } => tracing::warn!(
512 event = "attempt_retry",
513 attempt_id = %attempt,
514 reason = operation,
515 retry_in_ms = millis(delay),
516 ),
517 AttemptEvent::Launched {
518 attempt,
519 started,
520 timings,
521 } => {
522 macro_rules! launched {
523 ($level:ident, $name:literal) => {
524 tracing::$level!(
525 event = $name,
526 attempt_id = %attempt,
527 duration_ms = millis(timings.total()),
528 order_wait_ms = millis(timings.order_wait),
529 lock_wait_ms = millis(timings.lock_wait),
530 turn_wait_ms = millis(timings.turn_wait),
531 materialize_ms = millis(timings.materialize),
532 package_wait_ms = millis(timings.package.wait),
533 package_install_ms = millis(timings.package.install),
534 package_verify_ms = millis(timings.package.verify),
535 package_copy_ms = millis(timings.package.copy),
536 package_downloaded = timings.package.downloaded,
537 prepare_ms = millis(timings.prepare),
538 register_ms = millis(timings.register),
539 spawn_ms = millis(timings.spawn),
540 )
541 };
542 }
543 if !started {
544 launched!(warn, "attempt_launch_failed");
545 } else if timings.total() >= SLOW_LAUNCH {
546 launched!(warn, "attempt_launch_slow");
547 } else {
548 launched!(info, "attempt_launched");
549 }
550 }
551 _ => {}
552 }
553 }
554}
555
556fn millis(duration: Duration) -> u64 {
557 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
558}
559
560#[async_trait]
562pub trait DemandPersistence: fmt::Debug + Send + Sync {
563 async fn persists(&self, policy: PolicyId) -> bool;
564}
565
566#[derive(Debug, Clone, Copy, Default)]
567pub struct PersistentDemand;
568
569#[async_trait]
570impl DemandPersistence for PersistentDemand {
571 async fn persists(&self, _policy: PolicyId) -> bool {
572 true
573 }
574}
575
576#[async_trait]
577pub trait RetryDelay: fmt::Debug + Send + Sync {
578 async fn wait(&self, duration: Duration);
579}
580
581#[derive(Debug, Clone, Copy, Default)]
582pub struct TokioRetryDelay;
583
584#[async_trait]
585impl RetryDelay for TokioRetryDelay {
586 async fn wait(&self, duration: Duration) {
587 tokio::time::sleep(duration).await;
588 }
589}
590
591#[derive(Debug, Clone, PartialEq, Eq)]
592pub struct JitRequestFailure {
593 pub terminal: bool,
594 pub reason: FailureReason,
595 pub retry_after: Option<Duration>,
596}
597
598#[derive(Debug, Clone, Copy, PartialEq, Eq)]
602pub struct LifecycleGithubObservation {
603 pub status: GithubRunnerObservation,
604 pub runner_id: Option<u64>,
605}
606
607impl LifecycleGithubObservation {
608 #[must_use]
609 pub const fn unreachable() -> Self {
610 Self {
611 status: GithubRunnerObservation::Unreachable,
612 runner_id: None,
613 }
614 }
615
616 #[must_use]
617 pub const fn not_registered() -> Self {
618 Self {
619 status: GithubRunnerObservation::NotRegistered,
620 runner_id: None,
621 }
622 }
623
624 #[must_use]
625 pub const fn registered(runner_id: u64, busy: bool) -> Self {
626 Self {
627 status: GithubRunnerObservation::Registered { busy },
628 runner_id: Some(runner_id),
629 }
630 }
631}
632
633#[async_trait]
636pub trait LifecycleGithub: fmt::Debug + Send + Sync {
637 async fn register(
638 &self,
639 target: &ScaleTarget,
640 request: &JitRunnerRequest,
641 cancel: &CancelToken,
642 ) -> Result<JitRegistration, JitRequestFailure>;
643
644 async fn observe(
645 &self,
646 target: &ScaleTarget,
647 attempt: AttemptId,
648 cancel: &CancelToken,
649 ) -> LifecycleGithubObservation;
650
651 async fn deregister(&self, target: &ScaleTarget, runner_id: u64, cancel: &CancelToken) -> bool;
658}
659
660#[async_trait]
661impl<T> LifecycleGithub for T
662where
663 T: JitGateway + InventoryGateway + fmt::Debug + Send + Sync,
664{
665 async fn register(
666 &self,
667 target: &ScaleTarget,
668 request: &JitRunnerRequest,
669 cancel: &CancelToken,
670 ) -> Result<JitRegistration, JitRequestFailure> {
671 self.generate_jit_config(target, request, cancel)
672 .await
673 .map_err(|error| {
674 let reason = if matches!(&error, JitError::Forbidden { .. }) {
675 FailureReason::Other(
676 "GitHub refused JIT registration with 403; check the App's runner permission and runner-group access"
677 .into(),
678 )
679 } else {
680 FailureReason::JitRequestFailed
681 };
682 JitRequestFailure {
683 terminal: error.is_terminal(),
684 reason,
685 retry_after: error
686 .rate_limited()
687 .map(|limit| limit.delay_from(self.now())),
688 }
689 })
690 }
691
692 async fn observe(
693 &self,
694 target: &ScaleTarget,
695 attempt: AttemptId,
696 cancel: &CancelToken,
697 ) -> LifecycleGithubObservation {
698 let expected_name = runner_name(attempt);
699 match self.list_runners(target, cancel).await {
700 Ok(inventory) => inventory
701 .runners()
702 .iter()
703 .find(|runner| runner.name == expected_name)
704 .map_or(LifecycleGithubObservation::not_registered(), |runner| {
705 LifecycleGithubObservation::registered(runner.id, runner.busy)
706 }),
707 Err(_) => LifecycleGithubObservation::unreachable(),
708 }
709 }
710
711 async fn deregister(&self, target: &ScaleTarget, runner_id: u64, cancel: &CancelToken) -> bool {
712 self.remove_runner(target, runner_id, cancel).await.is_ok()
713 }
714}
715
716#[async_trait]
718pub trait RuntimePackages: fmt::Debug + Send + Sync {
719 async fn materialize(
722 &self,
723 attempt: &RunnerAttempt,
724 timings: &mut PackageTimings,
725 ) -> Result<RunnerVersion, FailureReason>;
726 fn release(&self, attempt: AttemptId) -> Result<(), FailureReason>;
727 fn prune_obsolete_guarded(
728 &self,
729 authority: PruneAuthority<'_>,
730 attempts: &[RunnerAttempt],
731 ) -> Result<(), FailureReason>;
732}
733
734pub struct PruneAuthority<'a> {
739 _guard: &'a AllocationGuard,
740}
741
742impl fmt::Debug for PruneAuthority<'_> {
743 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
744 f.write_str("PruneAuthority")
745 }
746}
747
748impl<'a> PruneAuthority<'a> {
749 fn from_launch_request(guard: &'a AllocationGuard) -> Self {
750 Self { _guard: guard }
751 }
752
753 #[cfg(test)]
754 pub(crate) const fn for_test(guard: &'a AllocationGuard) -> Self {
755 Self { _guard: guard }
756 }
757}
758
759#[derive(Debug, Clone, Default)]
766pub struct SharedPackages {
767 trees: Arc<tokio::sync::RwLock<()>>,
770 trusted: Arc<TrustedTrees>,
771 last_check: LastCheck,
772 failed_install: Arc<Mutex<Option<(std::time::Instant, FailureReason)>>>,
775}
776
777impl SharedPackages {
778 fn discard(
782 &self,
783 root: &Path,
784 origin: &Path,
785 contents: &Arc<PackageContents>,
786 reason: &'static str,
787 ) {
788 if self.trusted.forget_if_holding(root, contents) {
789 tracing::warn!(
790 event = "package_verify_failed",
791 reason,
792 "a shared runner package does not match its verified archive; it is discarded \
793 and prepared again"
794 );
795 self.trusted.forget(origin);
796 }
797 }
798
799 fn note_install(&self, result: Result<(), &FailureReason>) {
803 *self
804 .failed_install
805 .lock()
806 .unwrap_or_else(std::sync::PoisonError::into_inner) = result
807 .err()
808 .filter(|reason| !package_failure_is_terminal(reason))
809 .map(|reason| (std::time::Instant::now(), reason.clone()));
810 }
811
812 fn install_failed_since(&self, queued: std::time::Instant) -> Option<FailureReason> {
815 self.failed_install
816 .lock()
817 .unwrap_or_else(std::sync::PoisonError::into_inner)
818 .as_ref()
819 .filter(|(at, _)| *at >= queued)
820 .map(|(_, reason)| reason.clone())
821 }
822
823 #[must_use]
827 pub fn last_check(&self) -> LastCheck {
828 self.last_check.clone()
829 }
830}
831
832#[cfg(test)]
833impl SharedPackages {
834 pub(crate) async fn copying(&self) -> tokio::sync::RwLockReadGuard<'_, ()> {
835 self.trees.read().await
836 }
837
838 pub(crate) fn blind_to_edits_of(&self, root: &Path) {
841 let contents = self.trusted.contents_of(root);
842 self.trusted.record(root, contents).expect("recorded");
843 }
844
845 pub(crate) fn vouches_for(&self, root: &Path) -> bool {
846 self.trusted.contents_of(root).is_some()
847 }
848}
849
850#[derive(Debug)]
853pub struct CachedRuntimePackages {
854 cache: Arc<PackageCache>,
855 shared: SharedPackages,
856}
857
858async fn timed<T>(slot: &mut Duration, work: impl std::future::Future<Output = T>) -> T {
860 let began = std::time::Instant::now();
861 let result = work.await;
862 *slot += began.elapsed();
863 result
864}
865
866impl CachedRuntimePackages {
867 async fn blocking<T: Send + 'static>(
870 &self,
871 root: &Path,
872 work: impl FnOnce(&TrustedTrees, &Path) -> T + Send + 'static,
873 ) -> Result<T, FailureReason> {
874 let trusted = Arc::clone(&self.shared.trusted);
875 let root = root.to_path_buf();
876 tokio::task::spawn_blocking(move || work(&trusted, &root))
877 .await
878 .map_err(|_| FailureReason::ProcessStartFailed)
879 }
880
881 #[must_use]
883 pub const fn new(cache: Arc<PackageCache>, shared: SharedPackages) -> Self {
884 Self { cache, shared }
885 }
886
887 async fn reject_copy(
892 &self,
893 attempt: &RunnerAttempt,
894 root: &Path,
895 origin: &Path,
896 contents: &Arc<PackageContents>,
897 timings: &mut PackageTimings,
898 ) -> Result<(), FailureReason> {
899 let (attempt_copy, checking, from) =
900 (attempt.clone(), Arc::clone(contents), origin.to_path_buf());
901 let differs = timed(
902 &mut timings.verify,
903 self.blocking(root, move |_, source| {
904 let _ = remove_materialized_package(&attempt_copy);
905 let shared = checking.verify(source, |_| false);
906 shared.and_then(|()| {
907 if from == source {
908 Ok(())
909 } else {
910 checking.verify(&from, |_| false)
911 }
912 })
913 }),
914 )
915 .await?;
916 if let Err(mismatch) = differs {
917 self.shared
918 .discard(root, origin, contents, mismatch.as_str());
919 }
920 Ok(())
921 }
922
923 async fn ready(
927 &self,
928 destination: &Path,
929 timings: &mut PackageTimings,
930 ) -> Result<Option<(InstalledPackage, PathBuf)>, FailureReason> {
931 let Some(installed) = self.cache.reusable().map_err(package_failure)? else {
932 return Ok(None);
933 };
934 let destination = destination.to_path_buf();
935 let verifying = self.blocking(installed.root(), move |trusted, source| {
936 if trusted.holds(source) {
937 copy_origin_ready(source, &destination, trusted)
938 } else {
939 None
940 }
941 });
942 let origin = timed(&mut timings.verify, verifying).await?;
943 Ok(origin.map(|origin| (installed, origin)))
944 }
945
946 async fn install_trusted(
949 &self,
950 destination: &Path,
951 timings: &mut PackageTimings,
952 ) -> Result<(InstalledPackage, PathBuf), FailureReason> {
953 let (mut installed, downloaded) = timed(
954 &mut timings.install,
955 self.cache.ensure_installed_reporting(),
956 )
957 .await
958 .map_err(package_failure)?;
959 timings.downloaded |= downloaded;
960 let fresh = downloaded
965 || !timed(
966 &mut timings.verify,
967 self.blocking(installed.root(), TrustedTrees::holds),
968 )
969 .await?;
970 if fresh && !downloaded {
971 if self.shared.trusted.contents_of(installed.root()).is_some() {
973 tracing::warn!(
974 event = "package_verify_failed",
975 reason = "changed",
976 "a shared runner package changed since it was prepared; it is discarded and \
977 prepared again"
978 );
979 }
980 installed = timed(&mut timings.install, self.cache.reinstall())
981 .await
982 .map_err(package_failure)?;
983 timings.downloaded = true;
984 }
985 let destination = destination.to_path_buf();
986 let contents = installed.contents();
987 let origin = timed(
988 &mut timings.verify,
989 self.blocking(installed.root(), move |trusted, source| {
990 if fresh {
991 trusted.record(source, contents)?;
992 }
993 Ok::<_, std::io::Error>(copy_origin(source, &destination, trusted))
994 }),
995 )
996 .await?
997 .map_err(|_| FailureReason::ProcessStartFailed)?;
998 Ok((installed, origin))
999 }
1000}
1001
1002#[async_trait]
1003impl RuntimePackages for CachedRuntimePackages {
1004 async fn materialize(
1005 &self,
1006 attempt: &RunnerAttempt,
1007 timings: &mut PackageTimings,
1008 ) -> Result<RunnerVersion, FailureReason> {
1009 let destination = attempt.runtime_path().to_path_buf();
1010 let trees = &self.shared.trees;
1011 let mut installs = 0;
1017 let (hold, (installed, origin)) = loop {
1018 let queued = std::time::Instant::now();
1019 let hold = timed(&mut timings.wait, trees.read()).await;
1020 if let Some(ready) = self.ready(&destination, timings).await? {
1021 break (hold, ready);
1022 }
1023 drop(hold);
1024 let exclusive = timed(&mut timings.wait, trees.write()).await;
1030 if let Some(reason) = self.shared.install_failed_since(queued) {
1031 return Err(reason);
1032 }
1033 let installed = self.install_trusted(&destination, timings).await;
1034 self.shared.note_install(installed.as_ref().map(|_| ()));
1035 let ready = installed?;
1036 installs += 1;
1037 if installs == 2 {
1038 break (exclusive.downgrade(), ready);
1039 }
1040 };
1041 let Some(contents) = self.shared.trusted.contents_of(installed.root()) else {
1044 self.shared.trusted.forget(installed.root());
1045 return Err(FailureReason::ProcessStartFailed);
1046 };
1047 let copied_from = origin.clone();
1048 let copied = timed(
1051 &mut timings.copy,
1052 self.blocking(installed.root(), {
1053 let destination = destination.clone();
1054 move |trusted, source| {
1055 copy_package_from(source, &origin, &destination, trusted)?;
1056 if trusted.holds(source) {
1059 Ok(())
1060 } else {
1061 Err(std::io::Error::other(
1062 "the runner package changed while copied",
1063 ))
1064 }
1065 }
1066 }),
1067 )
1068 .await?;
1069 if copied.is_err() {
1070 drop(hold);
1073 self.reject_copy(attempt, installed.root(), &copied_from, &contents, timings)
1074 .await?;
1075 return Err(FailureReason::ProcessStartFailed);
1076 }
1077 if let Err(error) = self.cache.lease(attempt, installed.version()) {
1078 drop(hold);
1082 self.reject_copy(attempt, installed.root(), &copied_from, &contents, timings)
1083 .await?;
1084 return Err(package_failure(error));
1085 }
1086 drop(hold);
1089 let persistent = matches!(attempt.workspace(), AttemptWorkspace::PersistentSlot { .. });
1093 let checking = Arc::clone(&contents);
1094 let verified = timed(
1095 &mut timings.verify,
1096 self.blocking(installed.root(), move |_, _| {
1097 checking.verify(&destination, |name| persistent && is_work_folder(name))
1098 }),
1099 )
1100 .await?;
1101 if let Err(mismatch) = verified {
1102 let _ = self.cache.release(attempt.id);
1103 tracing::warn!(
1104 event = "package_copy_differs",
1105 attempt_id = %attempt.id,
1106 reason = mismatch.as_str(),
1107 "a runner's package copy does not match the package's verified archive; it is \
1108 being removed and will not be used"
1109 );
1110 self.reject_copy(attempt, installed.root(), &copied_from, &contents, timings)
1111 .await?;
1112 return Err(FailureReason::ProcessStartFailed);
1113 }
1114 Ok(installed.version().clone())
1115 }
1116
1117 fn release(&self, attempt: AttemptId) -> Result<(), FailureReason> {
1118 self.cache.release(attempt).map_err(package_failure)
1119 }
1120
1121 fn prune_obsolete_guarded(
1125 &self,
1126 _authority: PruneAuthority<'_>,
1127 attempts: &[RunnerAttempt],
1128 ) -> Result<(), FailureReason> {
1129 let Ok(_exclusive) = self.shared.trees.try_write() else {
1130 return Ok(());
1131 };
1132 let Some(current) = self.cache.newest_installed().map_err(package_failure)? else {
1133 return Ok(());
1134 };
1135 for installed in self.cache.installed().map_err(package_failure)? {
1136 if installed.version() != current.version() {
1137 match self.cache.prune(installed.version(), attempts) {
1138 Ok(()) => self.shared.trusted.forget(installed.root()),
1139 Err(PackageError::VersionInUse { .. }) => {}
1140 Err(error) => return Err(package_failure(error)),
1141 }
1142 }
1143 }
1144 Ok(())
1145 }
1146}
1147
1148fn package_failure(error: PackageError) -> FailureReason {
1149 error.failure_reason().unwrap_or(FailureReason::Other(
1150 "runner package cache operation failed".into(),
1151 ))
1152}
1153
1154fn package_failure_is_terminal(reason: &FailureReason) -> bool {
1155 matches!(
1156 reason,
1157 FailureReason::RunnerPackageUnverified | FailureReason::RunnerVersionRejected
1158 )
1159}
1160
1161const WORKSPACE_NAME_LEN: usize = 12;
1184
1185fn workspace_name(id: AttemptId) -> String {
1187 let full = id.to_string();
1188 full.chars()
1189 .filter(|c| *c != '-')
1190 .take(WORKSPACE_NAME_LEN)
1191 .collect()
1192}
1193
1194#[derive(Debug, Clone)]
1201struct Placement {
1202 runtime: PathBuf,
1203 workspace: AttemptWorkspace,
1204}
1205
1206fn root_failure(error: RunnerRootError) -> LifecycleError {
1213 tracing::warn!(
1230 error_kind = error.kind(),
1231 "the runner root refused this launch, so no attempt was created; the host will \
1232 retry every poll until the cause is resolved. Re-running `host set-runtime-root` \
1233 with the same path re-runs this check and prints the directory and the \
1234 remediation in full"
1235 );
1236 LifecycleError::Failed(FailureReason::Other(error.to_string()))
1237}
1238
1239fn lowest_free_slot(leases: &[RunnerAttempt], ceiling: NonZeroU16) -> Option<NonZeroU16> {
1248 let held: BTreeSet<u16> = leases
1249 .iter()
1250 .filter_map(|attempt| attempt.workspace().slot_number())
1251 .collect();
1252 (1..=ceiling.get())
1253 .find(|slot| !held.contains(slot))
1254 .and_then(NonZeroU16::new)
1255}
1256
1257fn create_or_validate_slot(slot: &Path) -> Result<(), LifecycleError> {
1265 match fs::symlink_metadata(slot) {
1266 Ok(metadata) if is_link_like(&metadata) => Err(slot_refusal(
1272 slot,
1273 "is a symbolic link, junction or other reparse point, which could place runner \
1274 files outside the configured root",
1275 )),
1276 Ok(metadata) if !metadata.is_dir() => Err(slot_refusal(slot, "is not a directory")),
1277 Ok(_) => Ok(()),
1278 Err(error) if error.kind() == std::io::ErrorKind::NotFound => fs::create_dir(slot)
1279 .map_err(|source| slot_refusal(slot, format!("could not be created: {source}"))),
1280 Err(source) => Err(slot_refusal(
1281 slot,
1282 format!("could not be inspected: {source}"),
1283 )),
1284 }
1285}
1286
1287fn accept_reusable_slot(slot: &Path) -> Result<(), LifecycleError> {
1299 let unreadable =
1300 |source: std::io::Error| slot_refusal(slot, format!("could not be read: {source}"));
1301 let entries = fs::read_dir(slot).map_err(unreadable)?;
1302 let mut refused: Vec<String> = Vec::new();
1303 for entry in entries {
1304 let entry = entry.map_err(unreadable)?;
1305 let name = entry.file_name();
1306 let metadata = fs::symlink_metadata(entry.path()).map_err(|source| {
1307 slot_refusal(
1308 slot,
1309 format!("entry {name:?} could not be inspected: {source}"),
1310 )
1311 })?;
1312 if is_retainable_work_folder(&name, &metadata) {
1317 continue;
1318 }
1319 refused.push(name.to_string_lossy().into_owned());
1320 }
1321 if refused.is_empty() {
1322 return Ok(());
1323 }
1324 refused.sort();
1325 Err(slot_refusal(
1326 slot,
1327 format!(
1328 "holds {} that this attempt may not reuse: [{}]. A reusable slot is empty or holds \
1329 one real `{DEFAULT_WORK_FOLDER}` directory and nothing else; remove or move the \
1330 entries listed, or let cleanup and recovery resolve them",
1331 if refused.len() == 1 {
1332 "an entry"
1333 } else {
1334 "entries"
1335 },
1336 refused.join(", ")
1337 ),
1338 ))
1339}
1340
1341fn slot_refusal(slot: &Path, detail: impl fmt::Display) -> LifecycleError {
1342 LifecycleError::Failed(FailureReason::Other(format!(
1343 "the persistent slot {} {detail}",
1344 slot.display()
1345 )))
1346}
1347
1348fn is_work_folder(name: &OsStr) -> bool {
1358 if cfg!(windows) {
1359 name.eq_ignore_ascii_case(DEFAULT_WORK_FOLDER)
1360 } else {
1361 name == OsStr::new(DEFAULT_WORK_FOLDER)
1362 }
1363}
1364
1365fn is_link_like(metadata: &fs::Metadata) -> bool {
1374 if metadata.file_type().is_symlink() {
1375 return true;
1376 }
1377 #[cfg(windows)]
1378 {
1379 use std::os::windows::fs::MetadataExt;
1380
1381 const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x0000_0400;
1382 metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0
1383 }
1384 #[cfg(not(windows))]
1385 false
1386}
1387
1388fn is_retainable_work_folder(name: &OsStr, metadata: &fs::Metadata) -> bool {
1396 is_work_folder(name) && metadata.is_dir() && !is_link_like(metadata)
1397}
1398
1399fn remove_materialized_package(attempt: &RunnerAttempt) -> std::io::Result<()> {
1406 match attempt.workspace() {
1407 AttemptWorkspace::Ephemeral => remove_runtime_tree(attempt.runtime_path()),
1408 AttemptWorkspace::PersistentSlot { .. } => scrub_slot_entries(attempt.runtime_path())
1409 .map_err(|quarantine| std::io::Error::other(quarantine.to_string())),
1410 }
1411}
1412
1413fn remove_runtime_tree(path: &Path) -> std::io::Result<()> {
1423 #[cfg(windows)]
1424 {
1425 match remove_dir_all::remove_dir_all(path) {
1426 Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
1427 unlink_reparse_points(path);
1428 remove_dir_all::remove_dir_all(path)
1429 }
1430 other => other,
1431 }
1432 }
1433 #[cfg(not(windows))]
1434 {
1435 fs::remove_dir_all(path)
1436 }
1437}
1438
1439#[cfg(windows)]
1459fn unlink_reparse_points(dir: &Path) {
1460 let Ok(entries) = fs::read_dir(dir) else {
1461 return;
1462 };
1463 for entry in entries.flatten() {
1464 let path = entry.path();
1465 let Ok(metadata) = entry.metadata() else {
1466 continue;
1467 };
1468 if is_link_like(&metadata) {
1469 let _ = fs::remove_dir(&path).or_else(|_| fs::remove_file(&path));
1470 } else if metadata.is_dir() {
1471 unlink_reparse_points(&path);
1472 }
1473 }
1474}
1475
1476#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1491enum SlotRefusal {
1492 NotTheJournalledSlot,
1494 PolicyRootDisagrees,
1496 Containment,
1498 SlotNotADirectory,
1500 Enumeration,
1502 WorkNotADirectory,
1504 Deletion,
1506 Residue,
1508}
1509
1510impl SlotRefusal {
1511 const fn class(self) -> &'static str {
1513 match self {
1514 Self::NotTheJournalledSlot => "slot_path_is_not_the_journalled_slot",
1515 Self::PolicyRootDisagrees => "slot_root_disagrees_with_policy",
1516 Self::Containment => "slot_escapes_its_root",
1517 Self::SlotNotADirectory => "slot_is_not_a_directory",
1518 Self::Enumeration => "slot_could_not_be_enumerated",
1519 Self::WorkNotADirectory => "retained_work_is_not_a_directory",
1520 Self::Deletion => "slot_entry_could_not_be_removed",
1521 Self::Residue => "slot_still_holds_runner_state",
1522 }
1523 }
1524
1525 const fn remediation(self) -> &'static str {
1527 match self {
1528 Self::NotTheJournalledSlot | Self::PolicyRootDisagrees | Self::Containment => {
1529 "the attempt keeps its slot lease and nothing was removed; correct the \
1530 repository's persistent workspace path, or remove the slot directory by hand \
1531 once you have confirmed what is in it"
1532 }
1533 Self::SlotNotADirectory | Self::WorkNotADirectory => {
1534 "the attempt keeps its slot lease and nothing was removed; a job replaced the \
1535 slot or its `_work` with a link, so inspect it before deleting anything and \
1536 treat the retained workspace as untrusted"
1537 }
1538 Self::Enumeration | Self::Deletion | Self::Residue => {
1539 "the attempt keeps its slot lease and will be cleaned again on the next pass; \
1540 release whatever is holding the files open, or remove the slot's contents by \
1541 hand leaving only `_work`"
1542 }
1543 }
1544 }
1545}
1546
1547#[derive(Debug, Clone, PartialEq, Eq)]
1554struct SlotQuarantine {
1555 refusal: SlotRefusal,
1556 detail: String,
1557}
1558
1559impl SlotQuarantine {
1560 fn new(refusal: SlotRefusal, detail: impl Into<String>) -> Self {
1561 Self {
1562 refusal,
1563 detail: detail.into(),
1564 }
1565 }
1566}
1567
1568impl fmt::Display for SlotQuarantine {
1569 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1570 write!(f, "{}; {}", self.detail, self.refusal.remediation())
1571 }
1572}
1573
1574fn verify_journalled_slot(
1588 runtime: &Path,
1589 slot: NonZeroU16,
1590 configured: Option<&LocalAbsolutePath>,
1591) -> Result<(), SlotQuarantine> {
1592 let mislaid = || {
1593 SlotQuarantine::new(
1594 SlotRefusal::NotTheJournalledSlot,
1595 format!(
1596 "the journalled runtime {} is not the slot s{slot} this attempt was allocated as",
1597 runtime.display()
1598 ),
1599 )
1600 };
1601 let local = |path: &Path| {
1602 path.to_str()
1603 .and_then(|raw| LocalAbsolutePath::new(raw).ok())
1604 .ok_or_else(mislaid)
1605 };
1606
1607 let runtime_path = local(runtime)?;
1608 let root = local(runtime.parent().ok_or_else(mislaid)?)?;
1609 let name = AttemptWorkspace::persistent_slot(slot)
1616 .slot_directory_name()
1617 .expect("a persistent workspace names its slot directory");
1618 let derived = runner_root::derive_child(&root, &name).map_err(|_| mislaid())?;
1619 if derived != runtime_path {
1620 return Err(mislaid());
1621 }
1622 if let Some(configured) = configured
1623 && configured != &root
1624 {
1625 return Err(SlotQuarantine::new(
1626 SlotRefusal::PolicyRootDisagrees,
1627 format!(
1628 "the journalled slot {} is not under the repository's configured persistent root \
1629 {}",
1630 runtime.display(),
1631 configured.as_str()
1632 ),
1633 ));
1634 }
1635 runner_root::verify_containment(&root, &derived).map_err(|source| {
1638 SlotQuarantine::new(
1639 SlotRefusal::Containment,
1640 format!("the journalled slot is not inside the root it was allocated from: {source}"),
1641 )
1642 })
1643}
1644
1645fn slot_is_present(slot: &Path) -> Result<bool, SlotQuarantine> {
1656 match fs::symlink_metadata(slot) {
1657 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
1658 Err(source) => Err(SlotQuarantine::new(
1659 SlotRefusal::SlotNotADirectory,
1660 format!(
1661 "the slot {} could not be inspected: {:?}",
1662 slot.display(),
1663 source.kind()
1664 ),
1665 )),
1666 Ok(metadata) if !metadata.is_dir() || is_link_like(&metadata) => Err(SlotQuarantine::new(
1667 SlotRefusal::SlotNotADirectory,
1668 format!(
1669 "the slot {} is a link or a file rather than a real directory",
1670 slot.display()
1671 ),
1672 )),
1673 Ok(_) => Ok(true),
1674 }
1675}
1676
1677fn scrub_slot_entries(slot: &Path) -> Result<(), SlotQuarantine> {
1692 let unreadable = |source: std::io::Error| {
1693 SlotQuarantine::new(
1694 SlotRefusal::Enumeration,
1695 format!(
1696 "the entries of {} could not be listed: {:?}",
1697 slot.display(),
1698 source.kind()
1699 ),
1700 )
1701 };
1702 for entry in fs::read_dir(slot).map_err(unreadable)? {
1703 let name = entry.map_err(unreadable)?.file_name();
1704 let path = slot.join(&name);
1705 let Some(metadata) = listed_entry_metadata(&path).map_err(unreadable)? else {
1708 continue;
1709 };
1710 if is_work_folder(&name) {
1711 if is_retainable_work_folder(&name, &metadata) {
1712 continue;
1713 }
1714 return Err(SlotQuarantine::new(
1715 SlotRefusal::WorkNotADirectory,
1716 format!(
1717 "the retained `{DEFAULT_WORK_FOLDER}` in {} is a link or a file rather than a \
1718 real directory",
1719 slot.display()
1720 ),
1721 ));
1722 }
1723 remove_slot_entry(&path, &metadata).map_err(|source| {
1724 SlotQuarantine::new(
1725 SlotRefusal::Deletion,
1726 format!(
1727 "an entry of {} could not be removed: {:?}",
1728 slot.display(),
1729 source.kind()
1730 ),
1731 )
1732 })?;
1733 }
1734 Ok(())
1735}
1736
1737fn listed_entry_metadata(path: &Path) -> std::io::Result<Option<fs::Metadata>> {
1743 match fs::symlink_metadata(path) {
1744 Ok(metadata) => Ok(Some(metadata)),
1745 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
1746 Err(source) => Err(source),
1747 }
1748}
1749
1750fn remove_slot_entry(path: &Path, metadata: &fs::Metadata) -> std::io::Result<()> {
1755 let removed = if is_link_like(metadata) {
1756 fs::remove_file(path).or_else(|_| fs::remove_dir(path))
1759 } else if metadata.is_dir() {
1760 remove_runtime_tree(path)
1761 } else {
1762 fs::remove_file(path)
1763 };
1764 match removed {
1765 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
1766 other => other,
1767 }
1768}
1769
1770fn verify_slot_scrubbed(slot: &Path) -> Result<(), SlotQuarantine> {
1787 let unreadable = |source: std::io::Error| {
1788 SlotQuarantine::new(
1789 SlotRefusal::Enumeration,
1790 format!(
1791 "the entries of {} could not be listed to verify the scrub: {:?}",
1792 slot.display(),
1793 source.kind()
1794 ),
1795 )
1796 };
1797 let mut residue = 0_usize;
1798 let mut named: Vec<String> = Vec::new();
1799 for entry in fs::read_dir(slot).map_err(unreadable)? {
1800 let name = entry.map_err(unreadable)?.file_name();
1801 let Some(metadata) = listed_entry_metadata(&slot.join(&name)).map_err(unreadable)? else {
1805 continue;
1806 };
1807 if is_retainable_work_folder(&name, &metadata) {
1808 continue;
1809 }
1810 residue = residue.saturating_add(1);
1811 if name
1812 .to_string_lossy()
1813 .starts_with(RestrictiveHandoff::NAME_PREFIX)
1814 {
1815 named.push("an encoded JIT handoff".to_owned());
1816 }
1817 }
1818 named.extend(
1819 SENSITIVE_SLOT_ENTRIES
1820 .iter()
1821 .filter(|entry| fs::symlink_metadata(slot.join(entry)).is_ok())
1822 .map(|entry| format!("`{entry}`")),
1823 );
1824 if residue == 0 && named.is_empty() {
1825 return Ok(());
1826 }
1827 named.sort_unstable();
1828 named.dedup();
1829 Err(SlotQuarantine::new(
1830 SlotRefusal::Residue,
1831 residue_detail(slot, residue, &named),
1832 ))
1833}
1834
1835fn residue_detail(slot: &Path, residue: usize, named: &[String]) -> String {
1845 if residue == 0 {
1846 format!(
1850 "the listing of {} reported nothing but `{DEFAULT_WORK_FOLDER}`, yet {} survived \
1851 cleanup",
1852 slot.display(),
1853 named.join(", ")
1854 )
1855 } else {
1856 format!(
1857 "{residue} entr{} other than `{DEFAULT_WORK_FOLDER}` survived cleanup of {}{}",
1858 if residue == 1 { "y" } else { "ies" },
1859 slot.display(),
1860 if named.is_empty() {
1861 String::new()
1862 } else {
1863 format!(", including {}", named.join(", "))
1864 }
1865 )
1866 }
1867}
1868
1869fn replacement_operation(outcome: &AttemptOutcome) -> Option<&'static str> {
1870 match outcome {
1871 AttemptOutcome::Failed {
1872 reason: FailureReason::JitExpired,
1873 } => Some("jit_expired_replacement"),
1874 AttemptOutcome::Failed {
1875 reason: FailureReason::ProcessExitedUnexpectedly,
1876 } => Some("exit_before_acceptance_replacement"),
1877 _ => None,
1878 }
1879}
1880
1881fn copy_origin(source: &Path, destination: &Path, seeds: &TrustedTrees) -> PathBuf {
1908 #[cfg(unix)]
1909 {
1910 package_seed(source, destination, seeds)
1911 .ok()
1912 .filter(|seed| refuse_work_folder(seed).is_ok())
1913 .unwrap_or_else(|| source.to_path_buf())
1914 }
1915 #[cfg(not(unix))]
1916 {
1917 let _ = (destination, seeds);
1918 source.to_path_buf()
1919 }
1920}
1921
1922fn copy_origin_ready(source: &Path, destination: &Path, seeds: &TrustedTrees) -> Option<PathBuf> {
1925 #[cfg(unix)]
1926 {
1927 match seed_location(source, destination) {
1928 Ok(None) => Some(source.to_path_buf()),
1929 Ok(Some(seeds_root)) => held_seed(source, &seeds_root, seeds)
1930 .ok()
1931 .flatten()
1932 .filter(|seed| refuse_work_folder(seed).is_ok()),
1933 Err(_) => None,
1934 }
1935 }
1936 #[cfg(not(unix))]
1937 {
1938 let _ = (destination, seeds);
1939 Some(source.to_path_buf())
1940 }
1941}
1942
1943fn copy_package_from(
1946 source: &Path,
1947 origin: &Path,
1948 destination: &Path,
1949 seeds: &TrustedTrees,
1950) -> std::io::Result<()> {
1951 refuse_work_folder(source)?;
1952
1953 #[cfg(unix)]
1954 {
1955 clone_tree(origin, destination)?;
1956 if origin != source && !seeds.holds(origin) {
1959 return Err(std::io::Error::other(
1960 "the package seed changed while cloned",
1961 ));
1962 }
1963 Ok(())
1964 }
1965 #[cfg(not(unix))]
1966 {
1967 let _ = (origin, seeds);
1968 copy_package_entries(source, destination, true)
1969 }
1970}
1971
1972fn refuse_work_folder(source: &Path) -> std::io::Result<()> {
1973 if source.join(DEFAULT_WORK_FOLDER).exists() {
1974 return Err(std::io::Error::new(
1975 std::io::ErrorKind::InvalidData,
1976 "a cached runner package contains a _work folder, which means it was used \
1977 to run a job before it was archived; the cache must only contain clean \
1978 extracts to prevent data leakage",
1979 ));
1980 }
1981 Ok(())
1982}
1983
1984#[cfg(unix)]
1986const PACKAGE_SEED_DIR: &str = ".runner-package";
1987
1988#[cfg(unix)]
2007fn package_seed(
2008 source: &Path,
2009 destination: &Path,
2010 seeds: &TrustedTrees,
2011) -> std::io::Result<PathBuf> {
2012 match seed_location(source, destination)? {
2013 Some(root) => refresh_package_seed(source, &root, seeds),
2014 None => Ok(source.to_path_buf()),
2015 }
2016}
2017
2018#[cfg(unix)]
2020fn seed_location(source: &Path, destination: &Path) -> std::io::Result<Option<PathBuf>> {
2021 use std::os::unix::fs::MetadataExt as _;
2022
2023 if !cfg!(target_os = "macos") {
2024 return Ok(None);
2025 }
2026 let root = destination
2027 .parent()
2028 .ok_or_else(|| std::io::Error::other("a runtime directory has no parent"))?;
2029 if fs::metadata(source)?.dev() == fs::metadata(root)?.dev() {
2030 return Ok(None);
2031 }
2032 Ok(Some(root.join(PACKAGE_SEED_DIR)))
2033}
2034
2035#[derive(Debug, Default)]
2052pub(crate) struct TrustedTrees {
2053 trees: Mutex<HashMap<PathBuf, TrustedTree>>,
2054}
2055
2056#[derive(Debug, Clone)]
2057struct TrustedTree {
2058 #[cfg(unix)]
2059 print: [u8; 32],
2060 contents: Option<Arc<PackageContents>>,
2061}
2062
2063impl TrustedTrees {
2064 fn trees(&self) -> std::sync::MutexGuard<'_, HashMap<PathBuf, TrustedTree>> {
2065 self.trees
2066 .lock()
2067 .unwrap_or_else(std::sync::PoisonError::into_inner)
2068 }
2069
2070 fn holds(&self, root: &Path) -> bool {
2073 let Some(tree) = self.trees().get(root).cloned() else {
2074 return false;
2075 };
2076 #[cfg(unix)]
2077 {
2078 tree_fingerprint(root).is_ok_and(|current| current == tree.print)
2079 }
2080 #[cfg(not(unix))]
2081 {
2082 let _ = tree;
2083 true
2084 }
2085 }
2086
2087 fn contents_of(&self, root: &Path) -> Option<Arc<PackageContents>> {
2089 self.trees()
2090 .get(root)
2091 .and_then(|tree| tree.contents.clone())
2092 }
2093
2094 fn record(&self, root: &Path, contents: Option<Arc<PackageContents>>) -> std::io::Result<()> {
2096 let tree = TrustedTree {
2097 #[cfg(unix)]
2098 print: tree_fingerprint(root)?,
2099 contents,
2100 };
2101 self.trees().insert(root.to_path_buf(), tree);
2102 Ok(())
2103 }
2104
2105 fn forget_if_holding(&self, root: &Path, contents: &Arc<PackageContents>) -> bool {
2108 let mut trees = self.trees();
2109 let holding = trees
2110 .get(root)
2111 .and_then(|tree| tree.contents.as_ref())
2112 .is_some_and(|current| Arc::ptr_eq(current, contents));
2113 if holding {
2114 trees.remove(root);
2115 }
2116 holding
2117 }
2118
2119 fn forget(&self, root: &Path) {
2121 self.trees().remove(root);
2122 }
2123}
2124
2125#[cfg(unix)]
2128fn tree_fingerprint(root: &Path) -> std::io::Result<[u8; 32]> {
2129 use sha2::{Digest as _, Sha256};
2130 use std::os::unix::fs::MetadataExt as _;
2131
2132 fn walk(root: &Path, path: &Path, hasher: &mut Sha256) -> std::io::Result<()> {
2133 let metadata = fs::symlink_metadata(path)?;
2134 let relative = path.strip_prefix(root).unwrap_or(path);
2135 hasher.update(relative.as_os_str().as_encoded_bytes());
2136 hasher.update([0]);
2137 for field in [
2138 metadata.ino(),
2139 metadata.size(),
2140 u64::from(metadata.mode()),
2141 u64::from(metadata.uid()),
2142 u64::from(metadata.gid()),
2143 ] {
2144 hasher.update(field.to_le_bytes());
2145 }
2146 hasher.update(metadata.ctime().to_le_bytes());
2147 hasher.update(metadata.ctime_nsec().to_le_bytes());
2148 if metadata.is_dir() {
2149 let mut children = fs::read_dir(path)?
2150 .map(|entry| entry.map(|entry| entry.path()))
2151 .collect::<std::io::Result<Vec<_>>>()?;
2152 children.sort();
2153 for child in children {
2154 walk(root, &child, hasher)?;
2155 }
2156 }
2157 Ok(())
2158 }
2159
2160 let mut hasher = Sha256::new();
2161 walk(root, root, &mut hasher)?;
2162 Ok(hasher.finalize().into())
2163}
2164
2165#[cfg(unix)]
2166fn seed_path(source: &Path, seeds: &Path) -> std::io::Result<PathBuf> {
2167 let version = source
2168 .file_name()
2169 .ok_or_else(|| std::io::Error::other("a cache entry has no name"))?;
2170 Ok(seeds.join(version))
2171}
2172
2173#[cfg(unix)]
2176fn held_seed(
2177 source: &Path,
2178 seeds: &Path,
2179 trusted: &TrustedTrees,
2180) -> std::io::Result<Option<PathBuf>> {
2181 use crate::package::MANIFEST_FILE;
2182
2183 let seed = seed_path(source, seeds)?;
2184 let manifest = fs::read(source.join(MANIFEST_FILE))?;
2185 let held = fs::read(seed.join(MANIFEST_FILE)).is_ok_and(|held| held == manifest)
2186 && trusted.holds(&seed);
2187 Ok(held.then_some(seed))
2188}
2189
2190#[cfg(unix)]
2192fn refresh_package_seed(
2193 source: &Path,
2194 seeds: &Path,
2195 trusted: &TrustedTrees,
2196) -> std::io::Result<PathBuf> {
2197 if let Some(seed) = held_seed(source, seeds, trusted)? {
2198 return Ok(seed);
2199 }
2200 let seed = seed_path(source, seeds)?;
2201
2202 match remove_runtime_tree(seeds) {
2204 Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error),
2205 _ => {}
2206 }
2207 fs::create_dir_all(seeds)?;
2208 let staging = seeds.join(format!(".staging-{}", uuid::Uuid::new_v4()));
2209 fs::create_dir(&staging)?;
2210 if let Err(error) = clone_tree(source, &staging).and_then(|()| fs::rename(&staging, &seed)) {
2212 let _ = remove_runtime_tree(&staging);
2213 return Err(error);
2214 }
2215 trusted.record(&seed, None)?;
2217 Ok(seed)
2218}
2219
2220#[cfg(unix)]
2225fn clone_tree(source: &Path, destination: &Path) -> std::io::Result<()> {
2226 #[cfg(target_os = "macos")]
2227 const CLONE: &[&str] = &["-c"];
2228 #[cfg(target_os = "linux")]
2229 const CLONE: &[&str] = &["--reflink=auto"];
2230 #[cfg(not(any(target_os = "macos", target_os = "linux")))]
2231 const CLONE: &[&str] = &[];
2232
2233 let from = format!("{}/.", source.display());
2234 let cp = |extra: &[&str]| {
2235 std::process::Command::new("cp")
2236 .arg("-a")
2237 .args(extra)
2238 .arg(&from)
2239 .arg(destination)
2240 .status()
2241 };
2242 if cp(CLONE)?.success() || cp(&[])?.success() {
2243 Ok(())
2244 } else {
2245 Err(std::io::Error::other("cp failed"))
2246 }
2247}
2248
2249#[cfg(not(unix))]
2250fn copy_package_entries(source: &Path, destination: &Path, top_level: bool) -> std::io::Result<()> {
2251 fs::create_dir_all(destination)?;
2252 for entry in fs::read_dir(source)? {
2253 let entry = entry?;
2254 if top_level && is_work_folder(&entry.file_name()) {
2255 return Err(std::io::Error::new(
2256 std::io::ErrorKind::InvalidData,
2257 format!(
2258 "the runner package holds a top-level `{DEFAULT_WORK_FOLDER}`; copying \
2259 it would overwrite the job workspace a persistent slot retains"
2260 ),
2261 ));
2262 }
2263 let target = destination.join(entry.file_name());
2264 if entry.file_type()?.is_dir() {
2265 copy_package_entries(&entry.path(), &target, false)?;
2266 } else {
2267 fs::copy(entry.path(), target)?;
2268 }
2269 }
2270 Ok(())
2271}
2272
2273#[derive(Debug, Clone, PartialEq, Eq)]
2276pub struct ProcessStartFailure {
2277 pub reason: FailureReason,
2278 pub retryable: bool,
2281 pub live_pid: Option<u32>,
2284}
2285
2286impl ProcessStartFailure {
2287 pub(crate) fn before_spawn(reason: FailureReason) -> Self {
2288 Self {
2289 reason,
2290 retryable: true,
2291 live_pid: None,
2292 }
2293 }
2294
2295 fn after_spawn_stopped() -> Self {
2296 Self {
2297 reason: FailureReason::ProcessStartFailed,
2298 retryable: false,
2299 live_pid: None,
2300 }
2301 }
2302
2303 fn after_spawn_live(pid: u32) -> Self {
2304 Self::after_spawn_live_with_reason(pid, FailureReason::ProcessStartFailed)
2305 }
2306
2307 fn after_spawn_live_with_reason(pid: u32, reason: FailureReason) -> Self {
2308 Self {
2309 reason,
2310 retryable: false,
2311 live_pid: Some(pid),
2312 }
2313 }
2314}
2315
2316#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2319pub enum ProviderCapability {
2320 Ready,
2321 Unsupported,
2322 NotInstalled,
2323 PermissionDenied,
2324 ImageUnavailableOrIncompatible,
2325 DiskQuotaUnavailable,
2326 Degraded,
2327}
2328
2329impl ProviderCapability {
2330 #[must_use]
2331 pub const fn display_name(self) -> &'static str {
2332 match self {
2333 Self::Ready => "ready",
2334 Self::Unsupported => "unsupported",
2335 Self::NotInstalled => "not installed",
2336 Self::PermissionDenied => "permission denied",
2337 Self::ImageUnavailableOrIncompatible => "image unavailable or incompatible",
2338 Self::DiskQuotaUnavailable => "disk quota unavailable",
2339 Self::Degraded => "degraded",
2340 }
2341 }
2342
2343 pub(crate) fn refusal(self) -> FailureReason {
2344 FailureReason::IsolationProvider(match self {
2345 Self::Ready | Self::Degraded => IsolationProviderFailure::Degraded,
2346 Self::Unsupported => IsolationProviderFailure::Unsupported,
2347 Self::NotInstalled => IsolationProviderFailure::NotInstalled,
2348 Self::PermissionDenied => IsolationProviderFailure::PermissionDenied,
2349 Self::ImageUnavailableOrIncompatible => {
2350 IsolationProviderFailure::ImageUnavailableOrIncompatible
2351 }
2352 Self::DiskQuotaUnavailable => IsolationProviderFailure::DiskQuotaUnavailable,
2353 })
2354 }
2355}
2356
2357fn safe_isolation_reason(
2360 reason: FailureReason,
2361 fallback: IsolationProviderFailure,
2362) -> FailureReason {
2363 match reason {
2364 FailureReason::IsolationProvider(_) => reason,
2365 _ => FailureReason::IsolationProvider(fallback),
2366 }
2367}
2368
2369#[derive(Debug, Clone, PartialEq, Eq)]
2370pub struct PreparedEnvironment {
2371 attempt: AttemptId,
2372 identity: Option<EnvironmentIdentity>,
2373 cache_target: Option<runner_manager_domain::model::ScaleTarget>,
2376}
2377
2378impl PreparedEnvironment {
2379 #[must_use]
2381 pub fn isolated(attempt: AttemptId, identity: EnvironmentIdentity) -> Self {
2382 Self {
2383 attempt,
2384 identity: Some(identity),
2385 cache_target: None,
2386 }
2387 }
2388
2389 #[must_use]
2390 pub fn identity(&self) -> Option<&EnvironmentIdentity> {
2391 self.identity.as_ref()
2392 }
2393}
2394
2395#[derive(Debug, Clone, PartialEq, Eq)]
2396pub struct ResolvedEnvironment {
2397 pub provider_kind: Backend,
2398 pub image: ImageReference,
2399}
2400
2401#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2403pub enum ProviderDiagnostic {
2404 CapabilityUnavailable,
2405 ImageRejected,
2406 PrepareFailed,
2407 StartFailed,
2408 OwnershipMismatch,
2409 CleanupDeferred,
2410}
2411
2412#[derive(Debug, Clone, PartialEq, Eq)]
2413pub enum EnvironmentIdentity {
2414 NativeProcess(u32),
2415 Isolated {
2416 host: HostId,
2417 attempt: AttemptId,
2418 provider_kind: Backend,
2419 environment_id: String,
2420 resolved_image: ImageReference,
2421 generation: String,
2422 },
2423}
2424
2425#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2426pub enum EnvironmentState {
2427 Starting,
2428 Running,
2429 Exited,
2430 Missing,
2431}
2432
2433#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2434pub enum ProviderStop {
2435 Stopped,
2436 StillRunning,
2437}
2438
2439#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2440pub enum ProviderDestroy {
2441 Destroyed,
2442 Deferred(&'static str),
2443}
2444
2445fn identity_matches_intent(
2446 host: HostId,
2447 attempt: &RunnerAttempt,
2448 identity: &EnvironmentIdentity,
2449) -> bool {
2450 identity_same_allocation(host, attempt, identity)
2451 && match (attempt.execution(), identity) {
2452 (
2453 AttemptExecution::Isolated { environment_id, .. },
2454 EnvironmentIdentity::Isolated {
2455 environment_id: id, ..
2456 },
2457 ) => environment_id
2458 .as_ref()
2459 .is_none_or(|journal_id| journal_id == id),
2460 _ => false,
2461 }
2462}
2463
2464fn identity_same_allocation(
2465 host: HostId,
2466 attempt: &RunnerAttempt,
2467 identity: &EnvironmentIdentity,
2468) -> bool {
2469 match (attempt.execution(), identity) {
2470 (
2471 AttemptExecution::Isolated {
2472 provider_kind,
2473 environment_id: _,
2474 resolved_image,
2475 generation,
2476 },
2477 EnvironmentIdentity::Isolated {
2478 host: owner,
2479 attempt: owner_attempt,
2480 provider_kind: kind,
2481 environment_id: _,
2482 resolved_image: image,
2483 generation: resource_generation,
2484 },
2485 ) => {
2486 *owner == host
2487 && *owner_attempt == attempt.id
2488 && provider_kind == kind
2489 && resolved_image == image
2490 && generation == resource_generation
2491 }
2492 _ => false,
2493 }
2494}
2495
2496pub struct OneTimeJitHandoff<'a> {
2499 config: &'a EncodedJitConfig,
2500}
2501
2502impl fmt::Debug for OneTimeJitHandoff<'_> {
2503 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2504 f.write_str("OneTimeJitHandoff([redacted])")
2505 }
2506}
2507
2508impl<'a> OneTimeJitHandoff<'a> {
2509 pub(crate) fn new(config: &'a EncodedJitConfig) -> Self {
2510 Self { config }
2511 }
2512
2513 pub fn consume(self) -> &'a EncodedJitConfig {
2515 self.config
2516 }
2517}
2518
2519pub trait ExecutionProvider: fmt::Debug + Send + Sync {
2524 fn probe(&self, policy: &ScalePolicy) -> ProviderCapability {
2525 if policy.execution_policy().is_native() {
2526 ProviderCapability::Ready
2527 } else {
2528 ProviderCapability::Unsupported
2529 }
2530 }
2531
2532 fn resolve(&self, policy: &ScalePolicy) -> Result<Option<ResolvedEnvironment>, FailureReason> {
2534 if policy.execution_policy().is_native() {
2535 Ok(None)
2536 } else {
2537 Err(FailureReason::Other(
2538 "execution provider unavailable".into(),
2539 ))
2540 }
2541 }
2542
2543 fn prepare(
2544 &self,
2545 attempt: &RunnerAttempt,
2546 policy: &ScalePolicy,
2547 ) -> Result<PreparedEnvironment, FailureReason> {
2548 if !attempt.execution().is_native() || !policy.execution_policy().is_native() {
2549 return Err(FailureReason::Other(
2550 "execution provider unavailable".into(),
2551 ));
2552 }
2553 Ok(PreparedEnvironment {
2554 attempt: attempt.id,
2555 identity: None,
2556 cache_target: None,
2557 })
2558 }
2559
2560 fn start(
2561 &self,
2562 prepared: PreparedEnvironment,
2563 attempt: &RunnerAttempt,
2564 handoff: OneTimeJitHandoff<'_>,
2565 ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
2566 if prepared.attempt != attempt.id || !attempt.execution().is_native() {
2567 return Err(ProcessStartFailure::before_spawn(FailureReason::Other(
2568 "execution provider identity mismatch".into(),
2569 )));
2570 }
2571 self.spawn(attempt, handoff.consume())
2572 .map(EnvironmentIdentity::NativeProcess)
2573 }
2574
2575 fn inspect(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2576 if !attempt.execution().is_native() {
2577 return Err(FailureReason::Other(
2578 "execution provider unavailable".into(),
2579 ));
2580 }
2581 self.is_alive(attempt).map(|alive| {
2582 if alive {
2583 EnvironmentState::Running
2584 } else {
2585 EnvironmentState::Missing
2586 }
2587 })
2588 }
2589
2590 fn stop(&self, attempt: &RunnerAttempt) -> Result<ProviderStop, FailureReason> {
2591 if !attempt.execution().is_native() {
2592 return Err(FailureReason::Other(
2593 "execution provider unavailable".into(),
2594 ));
2595 }
2596 self.terminate(attempt).map(|()| ProviderStop::Stopped)
2597 }
2598
2599 fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
2600 if !attempt.execution().is_native() {
2601 return Err(FailureReason::Other(
2602 "execution provider unavailable".into(),
2603 ));
2604 }
2605 Ok(ProviderDestroy::Destroyed)
2610 }
2611
2612 fn recover(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2613 self.inspect(attempt)
2614 }
2615
2616 fn enumerate_owned(&self, _host_id: HostId) -> Vec<EnvironmentIdentity> {
2617 Vec::new() }
2619
2620 fn diagnostics(&self, _attempt: &RunnerAttempt) -> Vec<ProviderDiagnostic> {
2621 Vec::new() }
2623
2624 fn owns(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
2626 Ok(attempt.execution().is_native())
2627 }
2628
2629 fn spawn(
2630 &self,
2631 attempt: &RunnerAttempt,
2632 config: &EncodedJitConfig,
2633 ) -> Result<u32, ProcessStartFailure>;
2634 fn is_alive(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason>;
2635 fn recovered_pid(&self, attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason>;
2638 fn completed_successfully(&self, attempt: &RunnerAttempt) -> bool;
2641 fn record_terminate_intent(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason>;
2642 fn has_terminate_intent(&self, attempt: &RunnerAttempt) -> bool;
2643 fn terminate(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason>;
2644}
2645
2646pub use ExecutionProvider as ProcessSupervisor;
2648
2649#[derive(Debug, Default)]
2652pub struct NativeProcesses {
2653 children: Mutex<BTreeMap<AttemptId, ChildProcess>>,
2654 successful_exits: Mutex<BTreeMap<AttemptId, bool>>,
2655 runner_env_file: Option<PathBuf>,
2658 dependency_caches: Option<DependencyCaches>,
2660 docker_cleanup: Option<crate::docker_cleanup::DockerCleanup>,
2661 runner: RunnerIdentity,
2663 assignments: Mutex<BTreeMap<AttemptId, RunnerAccount>>,
2665 #[cfg(test)]
2666 post_spawn_faults: Mutex<VecDeque<PostSpawnBoundary>>,
2667 #[cfg(test)]
2668 post_spawn_reaps: std::sync::atomic::AtomicUsize,
2669 #[cfg(test)]
2670 post_spawn_stop_failures: std::sync::atomic::AtomicUsize,
2671 #[cfg(test)]
2672 use_long_lived_test_listener: std::sync::atomic::AtomicBool,
2673}
2674
2675#[derive(Debug)]
2679pub struct PlatformExecutionProvider {
2680 native: NativeProcesses,
2681 oci: OciProcesses,
2682 windows_hyper_v: WindowsHyperVContainers,
2683}
2684
2685impl PlatformExecutionProvider {
2686 #[must_use]
2687 pub fn new(host_id: HostId) -> Self {
2688 Self {
2689 native: NativeProcesses::new(),
2690 oci: OciProcesses::new(host_id),
2691 windows_hyper_v: WindowsHyperVContainers::new(host_id),
2692 }
2693 }
2694
2695 #[must_use]
2697 pub fn with_isolation_settings(mut self, settings: crate::oci::IsolationSettings) -> Self {
2698 self.oci.apply(settings);
2699 self
2700 }
2701
2702 #[must_use]
2705 pub fn with_runner_env_file(mut self, path: PathBuf) -> Self {
2706 self.native = self.native.with_runner_env_file(path);
2707 self
2708 }
2709
2710 #[must_use]
2713 pub fn with_dependency_caches(mut self, caches: DependencyCaches) -> Self {
2714 self.native = self.native.with_dependency_caches(caches);
2715 self
2716 }
2717
2718 #[must_use]
2719 pub fn with_docker_cleanup(mut self, cleanup: crate::docker_cleanup::DockerCleanup) -> Self {
2720 self.native.docker_cleanup = Some(cleanup);
2721 self
2722 }
2723
2724 #[must_use]
2728 pub fn with_runner_pool(mut self, pool: Result<Option<RunnerPool>, String>) -> Self {
2729 self.native.runner = match pool {
2730 Ok(None) => RunnerIdentity::Daemon,
2731 Ok(Some(pool)) => {
2732 self.native.dependency_caches = self
2733 .native
2734 .dependency_caches
2735 .take()
2736 .map(DependencyCaches::without_git);
2737 RunnerIdentity::Pool(pool)
2738 }
2739 Err(why) => {
2740 tracing::error!(reason = "runner_account_missing", "{why}");
2741 RunnerIdentity::Missing(why)
2742 }
2743 };
2744 self
2745 }
2746
2747 fn isolated_provider(&self, backend: Backend) -> Option<&dyn ExecutionProvider> {
2748 match backend {
2749 Backend::Auto => {
2750 #[cfg(target_os = "windows")]
2751 {
2752 Some(&self.windows_hyper_v)
2753 }
2754 #[cfg(not(target_os = "windows"))]
2755 {
2756 Some(&self.oci)
2757 }
2758 }
2759 Backend::Oci => Some(&self.oci),
2760 Backend::WindowsHyperVContainer => Some(&self.windows_hyper_v),
2761 Backend::VirtualMachine => None,
2762 }
2763 }
2764
2765 fn provider_for_attempt(&self, attempt: &RunnerAttempt) -> Option<&dyn ExecutionProvider> {
2766 match attempt.execution() {
2767 AttemptExecution::Native { .. } => Some(&self.native),
2768 AttemptExecution::Isolated { provider_kind, .. } => {
2769 self.isolated_provider(*provider_kind)
2770 }
2771 }
2772 }
2773
2774 fn required_provider_for_attempt(
2775 &self,
2776 attempt: &RunnerAttempt,
2777 ) -> Result<&dyn ExecutionProvider, FailureReason> {
2778 self.provider_for_attempt(attempt)
2779 .ok_or_else(|| FailureReason::Other("execution provider unavailable".into()))
2780 }
2781}
2782
2783impl ExecutionProvider for PlatformExecutionProvider {
2784 fn probe(&self, policy: &ScalePolicy) -> ProviderCapability {
2785 match policy.execution_policy() {
2786 runner_manager_domain::execution::ExecutionPolicy::Native => self.native.probe(policy),
2787 runner_manager_domain::execution::ExecutionPolicy::Isolated { backend, .. } => self
2788 .isolated_provider(*backend)
2789 .map_or(ProviderCapability::Unsupported, |provider| {
2790 provider.probe(policy)
2791 }),
2792 }
2793 }
2794
2795 fn resolve(&self, policy: &ScalePolicy) -> Result<Option<ResolvedEnvironment>, FailureReason> {
2796 match policy.execution_policy() {
2797 runner_manager_domain::execution::ExecutionPolicy::Native => {
2798 self.native.resolve(policy)
2799 }
2800 runner_manager_domain::execution::ExecutionPolicy::Isolated { backend, .. } => self
2801 .isolated_provider(*backend)
2802 .ok_or_else(|| FailureReason::Other("execution provider unavailable".into()))?
2803 .resolve(policy),
2804 }
2805 }
2806
2807 fn prepare(
2808 &self,
2809 attempt: &RunnerAttempt,
2810 policy: &ScalePolicy,
2811 ) -> Result<PreparedEnvironment, FailureReason> {
2812 match policy.execution_policy() {
2813 runner_manager_domain::execution::ExecutionPolicy::Native => {
2814 self.native.prepare(attempt, policy)
2815 }
2816 runner_manager_domain::execution::ExecutionPolicy::Isolated { backend, .. } => self
2817 .isolated_provider(*backend)
2818 .ok_or_else(|| FailureReason::Other("execution provider unavailable".into()))?
2819 .prepare(attempt, policy),
2820 }
2821 }
2822
2823 fn start(
2824 &self,
2825 prepared: PreparedEnvironment,
2826 attempt: &RunnerAttempt,
2827 handoff: OneTimeJitHandoff<'_>,
2828 ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
2829 self.provider_for_attempt(attempt)
2830 .ok_or_else(|| {
2831 ProcessStartFailure::before_spawn(FailureReason::Other(
2832 "execution provider unavailable".into(),
2833 ))
2834 })?
2835 .start(prepared, attempt, handoff)
2836 }
2837
2838 fn inspect(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2839 self.required_provider_for_attempt(attempt)?
2840 .inspect(attempt)
2841 }
2842
2843 fn stop(&self, attempt: &RunnerAttempt) -> Result<ProviderStop, FailureReason> {
2844 self.required_provider_for_attempt(attempt)?.stop(attempt)
2845 }
2846
2847 fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
2848 self.required_provider_for_attempt(attempt)?
2849 .destroy(attempt)
2850 }
2851
2852 fn recover(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
2853 self.required_provider_for_attempt(attempt)?
2854 .recover(attempt)
2855 }
2856
2857 fn enumerate_owned(&self, host_id: HostId) -> Vec<EnvironmentIdentity> {
2858 let mut owned = self.oci.enumerate_owned(host_id);
2859 owned.extend(self.windows_hyper_v.enumerate_owned(host_id));
2860 owned
2861 }
2862
2863 fn diagnostics(&self, attempt: &RunnerAttempt) -> Vec<ProviderDiagnostic> {
2864 self.provider_for_attempt(attempt)
2865 .map_or_else(Vec::new, |provider| provider.diagnostics(attempt))
2866 }
2867
2868 fn owns(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
2869 self.required_provider_for_attempt(attempt)?.owns(attempt)
2870 }
2871
2872 fn spawn(
2873 &self,
2874 attempt: &RunnerAttempt,
2875 config: &EncodedJitConfig,
2876 ) -> Result<u32, ProcessStartFailure> {
2877 if !attempt.execution().is_native() {
2878 return Err(ProcessStartFailure::before_spawn(FailureReason::Other(
2879 "native process launch refused for an isolated allocation".into(),
2880 )));
2881 }
2882 self.native.spawn(attempt, config)
2883 }
2884
2885 fn is_alive(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
2886 self.required_provider_for_attempt(attempt)?
2887 .is_alive(attempt)
2888 }
2889
2890 fn recovered_pid(&self, attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
2891 self.required_provider_for_attempt(attempt)?
2892 .recovered_pid(attempt)
2893 }
2894
2895 fn completed_successfully(&self, attempt: &RunnerAttempt) -> bool {
2896 self.provider_for_attempt(attempt)
2897 .is_some_and(|provider| provider.completed_successfully(attempt))
2898 }
2899
2900 fn record_terminate_intent(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
2901 self.required_provider_for_attempt(attempt)?
2902 .record_terminate_intent(attempt)
2903 }
2904
2905 fn has_terminate_intent(&self, attempt: &RunnerAttempt) -> bool {
2906 self.provider_for_attempt(attempt)
2907 .is_some_and(|provider| provider.has_terminate_intent(attempt))
2908 }
2909
2910 fn terminate(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
2911 self.required_provider_for_attempt(attempt)?
2912 .terminate(attempt)
2913 }
2914}
2915
2916#[cfg(test)]
2917#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2918enum PostSpawnBoundary {
2919 HandoffDelete,
2920 IdentitySerialize,
2921 IdentityWrite,
2922 ChildMapInsert,
2923}
2924
2925impl NativeProcesses {
2926 #[must_use]
2927 pub fn new() -> Self {
2928 Self::default()
2929 }
2930
2931 #[must_use]
2933 pub fn with_runner_env_file(mut self, path: PathBuf) -> Self {
2934 self.runner_env_file = Some(path);
2935 self
2936 }
2937
2938 #[must_use]
2941 pub fn with_dependency_caches(mut self, caches: DependencyCaches) -> Self {
2942 self.dependency_caches = Some(match self.runner {
2943 RunnerIdentity::Pool(_) => caches.without_git(),
2944 RunnerIdentity::Daemon | RunnerIdentity::Missing(_) => caches,
2945 });
2946 self
2947 }
2948
2949 fn cache_env(
2951 &self,
2952 attempt: &RunnerAttempt,
2953 target: Option<&runner_manager_domain::model::ScaleTarget>,
2954 host_env: &RunnerEnv,
2955 ) -> Vec<(&'static str, std::ffi::OsString)> {
2956 match (&self.dependency_caches, target) {
2957 (Some(caches), Some(target)) => {
2958 caches.for_launch(target, attempt.runtime_path(), host_env)
2959 }
2960 _ => Vec::new(),
2961 }
2962 }
2963
2964 fn spawn_listener(
2967 &self,
2968 attempt: &RunnerAttempt,
2969 config: &EncodedJitConfig,
2970 cache_target: Option<&runner_manager_domain::model::ScaleTarget>,
2971 ) -> Result<u32, ProcessStartFailure> {
2972 let pool = self
2973 .runner
2974 .pool()
2975 .map_err(ProcessStartFailure::before_spawn)?;
2976 let account = self
2977 .account_for(attempt)
2978 .map_err(ProcessStartFailure::before_spawn)?;
2979 let handoff = RestrictiveHandoff::create(
2980 attempt.runtime_path(),
2981 SecretString::from(config.expose().to_owned()),
2982 )
2983 .map_err(|_| ProcessStartFailure::before_spawn(FailureReason::ProcessStartFailed))?;
2984 #[cfg(windows)]
2985 let program = attempt
2986 .runtime_path()
2987 .join("bin")
2988 .join("Runner.Listener.exe");
2989 #[cfg(not(windows))]
2990 let program = attempt.runtime_path().join("bin").join("Runner.Listener");
2991 if !program.is_file() {
2994 return Err(ProcessStartFailure::before_spawn(
2995 FailureReason::ProcessStartFailed,
2996 ));
2997 }
2998 let host_env = self.host_env().map_err(ProcessStartFailure::before_spawn)?;
2999 let caches = self.cache_env(attempt, cache_target, &host_env);
3000 let cache_names: Vec<&'static str> = caches.iter().map(|(name, _)| *name).collect();
3001 #[cfg(test)]
3002 let spec = if self
3003 .use_long_lived_test_listener
3004 .load(std::sync::atomic::Ordering::SeqCst)
3005 {
3006 SpawnSpec::new(program)
3007 .args([
3008 "--ignored",
3009 "--exact",
3010 "lifecycle::tests::long_lived_native_listener_helper",
3011 "--nocapture",
3012 ])
3013 .env(
3014 "RUNNER_MANAGER_TEST_LISTENER_READY",
3015 attempt.runtime_path().join(TEST_LISTENER_READY),
3016 )
3017 .working_dir(attempt.runtime_path())
3018 } else {
3019 runner_listener_spec(program, attempt.runtime_path(), caches, &host_env)
3020 };
3021 #[cfg(not(test))]
3022 let spec = runner_listener_spec(program, attempt.runtime_path(), caches, &host_env);
3023 let spec = match account.zip(pool) {
3024 Some((account, pool)) => {
3025 hand_over(account, pool, attempt.runtime_path(), &spec, &cache_names).map_err(
3026 |error| {
3027 tracing::warn!(attempt_id = %attempt.id, event = "runner_hand_over_failed", %error, "the attempt could not be handed to the runner account");
3028 ProcessStartFailure::before_spawn(FailureReason::ProcessStartFailed)
3029 },
3030 )?;
3031 spec.run_as(account)
3032 }
3033 None => spec,
3034 };
3035 let (spec, docker_proxy) = if let Some(cleanup) = &self.docker_cleanup {
3036 let repository = cache_target.map(ToString::to_string).unwrap_or_default();
3037 let listen =
3038 account.map(|account| (attempt.runtime_path().join(DOCKER_SOCKET_DIR), account));
3039 let own_directories: Vec<PathBuf> =
3042 std::iter::once(attempt.runtime_path().to_path_buf())
3043 .chain(cache_names.iter().filter_map(|name| {
3044 spec.configured_env(name)
3045 .map(PathBuf::from)
3046 .filter(|path| path.is_absolute())
3047 }))
3048 .collect();
3049 match cleanup.launch_environment(
3050 attempt.id,
3051 &repository,
3052 &host_env,
3053 listen,
3054 &own_directories,
3055 ) {
3056 Ok(Some(endpoint)) => (
3057 spec.env("DOCKER_HOST", endpoint),
3058 Some(cleanup.root.join(attempt.id.to_string())),
3059 ),
3060 Ok(None) => (spec, None),
3061 Err(error) => {
3062 tracing::warn!(attempt_id = %attempt.id, event = "docker_accounting_unavailable", reason = ?error.kind(), "native Docker accounting could not start");
3063 if crate::docker_cleanup::Config::load(&cleanup.config).map_or(true, |config| {
3064 config.mode == crate::docker_cleanup::Mode::Apply
3065 }) {
3066 return Err(ProcessStartFailure::before_spawn(
3067 FailureReason::ProcessStartFailed,
3068 ));
3069 }
3070 (spec, None)
3071 }
3072 }
3073 } else {
3074 (spec, None)
3075 };
3076 let child = spec
3077 .spawn_runner_with_handoff(&handoff)
3078 .map_err(|_| ProcessStartFailure::before_spawn(FailureReason::ProcessStartFailed))?;
3079 #[cfg(test)]
3081 if self.faults_at(PostSpawnBoundary::HandoffDelete) {
3082 drop(handoff);
3083 return Err(self.abort_spawned_child(child, attempt, false));
3084 }
3085 if handoff.delete().is_err() {
3086 return Err(self.abort_spawned_child(child, attempt, false));
3087 }
3088 #[cfg(test)]
3089 if self.faults_at(PostSpawnBoundary::IdentitySerialize) {
3090 return Err(self.abort_spawned_child(child, attempt, false));
3091 }
3092 let identity = match serde_json::to_vec(child.identity()) {
3093 Ok(identity) => identity,
3094 Err(_) => {
3095 return Err(self.abort_spawned_child(child, attempt, false));
3096 }
3097 };
3098 if self.persist_identity(attempt, &identity).is_err() {
3099 return Err(self.abort_spawned_child(child, attempt, true));
3100 }
3101 if let Some(directory) = docker_proxy
3102 && crate::docker_cleanup::write_json(&directory.join("runner.json"), child.identity())
3103 .is_err()
3104 {
3105 return Err(self.abort_spawned_child(child, attempt, true));
3106 }
3107 let pid = child.pid();
3108 #[cfg(test)]
3109 if self.faults_at(PostSpawnBoundary::ChildMapInsert) {
3110 return Err(self.abort_spawned_child(child, attempt, true));
3111 }
3112 let mut children = self
3113 .children
3114 .lock()
3115 .unwrap_or_else(std::sync::PoisonError::into_inner);
3116 children.insert(attempt.id, child);
3117 Ok(pid)
3118 }
3119
3120 fn host_env(&self) -> Result<RunnerEnv, FailureReason> {
3128 let Some(path) = &self.runner_env_file else {
3129 return Ok(RunnerEnv::default());
3130 };
3131 RunnerEnv::load(path).map_err(|error| {
3132 tracing::warn!(
3133 reason = "runner_env_file_invalid",
3134 "runner.env could not be applied, so no runner starts until it is fixed: {error}"
3135 );
3136 FailureReason::Other(format!(
3137 "runner.env cannot be applied ({error}); fix it with `runner-manager host env`"
3138 ))
3139 })
3140 }
3141
3142 #[cfg(test)]
3143 fn fail_post_spawn_at(&self, boundary: PostSpawnBoundary) {
3144 self.post_spawn_faults.lock().unwrap().push_back(boundary);
3145 }
3146
3147 #[cfg(test)]
3148 fn faults_at(&self, boundary: PostSpawnBoundary) -> bool {
3149 let mut faults = self.post_spawn_faults.lock().unwrap();
3150 if faults.front() == Some(&boundary) {
3151 faults.pop_front();
3152 true
3153 } else {
3154 false
3155 }
3156 }
3157
3158 #[cfg(test)]
3159 fn fail_post_spawn_stops(&self, count: usize) {
3160 self.post_spawn_stop_failures
3161 .fetch_add(count, std::sync::atomic::Ordering::SeqCst);
3162 }
3163
3164 #[cfg(test)]
3165 fn fail_next_post_spawn_stop(&self) {
3166 self.fail_post_spawn_stops(1);
3167 }
3168
3169 #[cfg(test)]
3170 fn use_long_lived_test_listener(&self) {
3171 self.use_long_lived_test_listener
3172 .store(true, std::sync::atomic::Ordering::SeqCst);
3173 }
3174
3175 fn stop_spawned_child(&self, child: &mut ChildProcess) -> Result<(), FailureReason> {
3176 #[cfg(test)]
3177 if self
3178 .post_spawn_stop_failures
3179 .fetch_update(
3180 std::sync::atomic::Ordering::SeqCst,
3181 std::sync::atomic::Ordering::SeqCst,
3182 |left| if left > 0 { Some(left - 1) } else { None },
3183 )
3184 .is_ok()
3185 {
3186 return Err(FailureReason::Other("injected runner stop failure".into()));
3187 }
3188 child
3189 .stop(Duration::from_secs(1))
3190 .map(|_| ())
3191 .map_err(|_| FailureReason::Other("spawned runner process could not be stopped".into()))
3192 }
3193
3194 fn abort_spawned_child(
3195 &self,
3196 mut child: ChildProcess,
3197 attempt: &RunnerAttempt,
3198 remove_identity: bool,
3199 ) -> ProcessStartFailure {
3200 let mut reaped = self.stop_spawned_child(&mut child).is_ok();
3201 if reaped {
3202 if remove_identity {
3203 Self::remove_identity_files(attempt);
3204 }
3205 } else {
3206 let identity_durable =
3209 serde_json::to_vec(child.identity())
3210 .ok()
3211 .is_some_and(|identity| {
3212 self.persist_identity(attempt, &identity).is_ok()
3213 || self.persist_fallback_identity(attempt, &identity).is_ok()
3214 });
3215 if !identity_durable {
3216 for _ in 1..MAX_POST_SPAWN_STOP_ATTEMPTS {
3221 if self.stop_spawned_child(&mut child).is_ok() {
3222 reaped = true;
3223 break;
3224 }
3225 }
3226 if !reaped {
3227 let pid = child.pid();
3232 let marker = write_durable_file(
3233 &Self::unresolved_process_path(attempt),
3234 pid.to_string().as_bytes(),
3235 );
3236 self.children
3237 .lock()
3238 .unwrap_or_else(std::sync::PoisonError::into_inner)
3239 .insert(attempt.id, child);
3240 let reason = if marker.is_ok() {
3241 FailureReason::Other(
3242 "spawn cleanup exhausted its bounded stop attempts; the live process remains under durable unresolved supervision"
3243 .into(),
3244 )
3245 } else {
3246 FailureReason::Other(
3247 "spawn cleanup exhausted its bounded stop attempts and the unresolved-process marker could not be journalled"
3248 .into(),
3249 )
3250 };
3251 return ProcessStartFailure::after_spawn_live_with_reason(pid, reason);
3252 }
3253 } else {
3254 let pid = child.pid();
3255 self.children
3256 .lock()
3257 .unwrap_or_else(std::sync::PoisonError::into_inner)
3258 .insert(attempt.id, child);
3259 return ProcessStartFailure::after_spawn_live(pid);
3260 }
3261 }
3262 #[cfg(test)]
3263 if reaped {
3264 self.post_spawn_reaps
3265 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
3266 }
3267 #[cfg(not(test))]
3268 let _ = reaped;
3269 ProcessStartFailure::after_spawn_stopped()
3270 }
3271
3272 fn identity_path(attempt: &RunnerAttempt) -> PathBuf {
3273 attempt.runtime_path().join(IDENTITY_FILE)
3274 }
3275
3276 fn fallback_identity_path(attempt: &RunnerAttempt) -> PathBuf {
3277 attempt.runtime_path().join(FALLBACK_IDENTITY_FILE)
3278 }
3279
3280 fn unresolved_process_path(attempt: &RunnerAttempt) -> PathBuf {
3281 attempt.runtime_path().join(UNRESOLVED_PROCESS_FILE)
3282 }
3283
3284 fn remove_identity_files(attempt: &RunnerAttempt) {
3285 let _ = fs::remove_file(Self::identity_path(attempt));
3286 let _ = fs::remove_file(Self::fallback_identity_path(attempt));
3287 let _ = fs::remove_file(Self::unresolved_process_path(attempt));
3288 }
3289
3290 fn persist_identity(&self, attempt: &RunnerAttempt, bytes: &[u8]) -> std::io::Result<()> {
3291 self.persist_identity_at(&Self::identity_path(attempt), bytes)
3292 }
3293
3294 fn persist_fallback_identity(
3295 &self,
3296 attempt: &RunnerAttempt,
3297 bytes: &[u8],
3298 ) -> std::io::Result<()> {
3299 self.persist_identity_at(&Self::fallback_identity_path(attempt), bytes)
3300 }
3301
3302 fn persist_identity_at(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
3303 #[cfg(test)]
3304 if self.faults_at(PostSpawnBoundary::IdentityWrite) {
3305 return Err(std::io::Error::other("injected identity write failure"));
3306 }
3307 write_durable_file(path, bytes)
3308 }
3309
3310 fn intent_path(attempt: &RunnerAttempt) -> PathBuf {
3311 attempt.runtime_path().join(TERMINATE_INTENT_FILE)
3312 }
3313
3314 fn read_identity(&self, attempt: &RunnerAttempt) -> Result<RecordedProcess, FailureReason> {
3323 let identity = match Self::read_identity_at(&Self::identity_path(attempt))? {
3324 Some(identity) => Some(identity),
3325 None => Self::read_identity_at(&Self::fallback_identity_path(attempt))?,
3326 };
3327 Ok(match identity {
3328 None => RecordedProcess::None,
3329 Some(identity) if self.runs_as_runner(attempt, &identity) => {
3330 RecordedProcess::Runner(identity)
3331 }
3332 Some(_) => RecordedProcess::Foreign,
3333 })
3334 }
3335
3336 fn runs_as_runner(&self, attempt: &RunnerAttempt, identity: &ProcessIdentity) -> bool {
3345 let RunnerIdentity::Pool(pool) = &self.runner else {
3346 return true;
3347 };
3348 let Some(account) = owner_in_pool(pool, attempt.runtime_path()) else {
3352 return false;
3353 };
3354 #[cfg(unix)]
3355 {
3356 use std::os::unix::fs::MetadataExt as _;
3357 match fs::metadata(format!("/proc/{}", identity.pid())) {
3358 Ok(metadata) => metadata.uid() == account.uid,
3359 Err(error) => error.kind() == std::io::ErrorKind::NotFound,
3360 }
3361 }
3362 #[cfg(not(unix))]
3363 {
3364 let _ = (account, identity);
3365 true
3366 }
3367 }
3368
3369 fn account_for(&self, attempt: &RunnerAttempt) -> Result<Option<RunnerAccount>, FailureReason> {
3381 let Some(pool) = self.runner.pool()? else {
3382 return Ok(None);
3383 };
3384 let mut assignments = self
3385 .assignments
3386 .lock()
3387 .unwrap_or_else(std::sync::PoisonError::into_inner);
3388 if let Some(account) = assignments.get(&attempt.id) {
3389 return Ok(Some(*account));
3390 }
3391 let runtime = attempt.runtime_path();
3392 let held = assignments.values().map(|account| account.uid).collect();
3393 let busy = || {
3396 runtime
3397 .parent()
3398 .and_then(|root| fs::read_dir(root).ok())
3399 .into_iter()
3400 .flatten()
3401 .flatten()
3402 .filter_map(|entry| owner_in_pool(pool, &entry.path()))
3403 .map(|account| account.uid)
3404 .collect()
3405 };
3406 let account = pool
3407 .allocate(owner_in_pool(pool, runtime), &held, busy)
3408 .ok_or_else(|| {
3409 FailureReason::Other(format!(
3410 "every one of the {} runner accounts is in use by another attempt, and no \
3411 two runners share one",
3412 pool.accounts.len()
3413 ))
3414 })?;
3415 assignments.insert(attempt.id, account);
3416 Ok(Some(account))
3417 }
3418
3419 fn read_identity_at(path: &Path) -> Result<Option<ProcessIdentity>, FailureReason> {
3420 match fs::read(path) {
3421 Ok(bytes) => serde_json::from_slice(&bytes)
3422 .map(Some)
3423 .map_err(|_| FailureReason::Other("process identity journal is unreadable".into())),
3424 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
3425 Err(_) => Err(FailureReason::Other(
3426 "process identity journal could not be read".into(),
3427 )),
3428 }
3429 }
3430}
3431
3432impl ProcessSupervisor for NativeProcesses {
3433 fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
3434 if !attempt.execution().is_native() {
3435 return Err(FailureReason::Other(
3436 "execution provider unavailable".into(),
3437 ));
3438 }
3439 if let Some(caches) = &self.dependency_caches {
3442 caches.capture_git(attempt.runtime_path());
3443 }
3444 self.assignments
3447 .lock()
3448 .unwrap_or_else(std::sync::PoisonError::into_inner)
3449 .remove(&attempt.id);
3450 Ok(ProviderDestroy::Destroyed)
3451 }
3452
3453 fn prepare(
3454 &self,
3455 attempt: &RunnerAttempt,
3456 policy: &ScalePolicy,
3457 ) -> Result<PreparedEnvironment, FailureReason> {
3458 if !attempt.execution().is_native() || !policy.execution_policy().is_native() {
3459 return Err(FailureReason::Other(
3460 "execution provider unavailable".into(),
3461 ));
3462 }
3463 self.host_env()?;
3464 if let Some(account) = self.account_for(attempt)? {
3469 hand_over_attempt(account, attempt.runtime_path()).map_err(|error| {
3470 tracing::warn!(attempt_id = %attempt.id, event = "runner_hand_over_failed", %error, "the attempt could not be handed to the runner account");
3471 FailureReason::ProcessStartFailed
3472 })?;
3473 }
3474 let cache_target = match &self.dependency_caches {
3477 Some(caches) => {
3478 caches.config()?;
3479 Some(policy.target.clone())
3480 }
3481 None => None,
3482 };
3483 Ok(PreparedEnvironment {
3484 attempt: attempt.id,
3485 identity: None,
3486 cache_target,
3487 })
3488 }
3489
3490 fn start(
3491 &self,
3492 prepared: PreparedEnvironment,
3493 attempt: &RunnerAttempt,
3494 handoff: OneTimeJitHandoff<'_>,
3495 ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
3496 if prepared.attempt != attempt.id || !attempt.execution().is_native() {
3497 return Err(ProcessStartFailure::before_spawn(FailureReason::Other(
3498 "execution provider identity mismatch".into(),
3499 )));
3500 }
3501 self.spawn_listener(attempt, handoff.consume(), prepared.cache_target.as_ref())
3502 .map(EnvironmentIdentity::NativeProcess)
3503 }
3504
3505 fn spawn(
3506 &self,
3507 attempt: &RunnerAttempt,
3508 config: &EncodedJitConfig,
3509 ) -> Result<u32, ProcessStartFailure> {
3510 self.spawn_listener(attempt, config, None)
3511 }
3512
3513 fn is_alive(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
3514 let mut children = self
3515 .children
3516 .lock()
3517 .unwrap_or_else(std::sync::PoisonError::into_inner);
3518 if let Some(child) = children.get_mut(&attempt.id) {
3519 return match child
3520 .try_exit_status()
3521 .map_err(|_| FailureReason::Other("runner process could not be observed".into()))?
3522 {
3523 None => Ok(true),
3524 Some(status) => {
3525 if let Ok(mut exits) = self.successful_exits.lock() {
3526 exits.insert(attempt.id, status.success());
3527 }
3528 Ok(false)
3529 }
3530 };
3531 }
3532 let identity = match self.read_identity(attempt)? {
3533 RecordedProcess::Runner(identity) => identity,
3534 RecordedProcess::Foreign => return Ok(false),
3535 RecordedProcess::None => {
3536 if attempt.process_id().is_some()
3537 || Self::unresolved_process_path(attempt).is_file()
3538 {
3539 return Err(FailureReason::Other(
3540 "runner process identity is missing; refusing recovery until the process is resolved"
3541 .into(),
3542 ));
3543 }
3544 return Ok(false);
3545 }
3546 };
3547 match identity.recheck() {
3548 Ok(Adoption::Live) => Ok(true),
3549 Ok(Adoption::Gone | Adoption::PidRecycled { .. }) => Ok(false),
3550 Err(_) => Ok(false),
3551 }
3552 }
3553
3554 fn recovered_pid(&self, attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
3555 Ok(self
3556 .read_identity(attempt)?
3557 .runner()
3558 .map(|identity| identity.pid()))
3559 }
3560
3561 fn completed_successfully(&self, attempt: &RunnerAttempt) -> bool {
3562 self.successful_exits
3563 .lock()
3564 .ok()
3565 .and_then(|exits| exits.get(&attempt.id).copied())
3566 .unwrap_or(false)
3567 }
3568
3569 fn record_terminate_intent(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
3570 let path = Self::intent_path(attempt);
3571 write_durable_file(&path, b"registration-timeout\n")
3572 .map_err(|_| FailureReason::Other("terminate intent could not be journalled".into()))
3573 }
3574
3575 fn has_terminate_intent(&self, attempt: &RunnerAttempt) -> bool {
3576 Self::intent_path(attempt).is_file()
3577 }
3578
3579 fn terminate(&self, attempt: &RunnerAttempt) -> Result<(), FailureReason> {
3580 let mut children = self
3581 .children
3582 .lock()
3583 .unwrap_or_else(std::sync::PoisonError::into_inner);
3584 if let Some(child) = children.get_mut(&attempt.id) {
3585 child
3586 .stop(Duration::from_secs(10))
3587 .map_err(|_| FailureReason::Other("runner process could not be stopped".into()))?;
3588 return Ok(());
3589 }
3590 let Some(identity) = self.read_identity(attempt)?.runner() else {
3591 return Ok(());
3592 };
3593 match identity
3594 .terminate(Duration::from_secs(10))
3595 .map_err(|_| FailureReason::Other("runner process could not be stopped".into()))?
3596 {
3597 Termination::Terminated | Termination::AlreadyGone => Ok(()),
3598 Termination::RefusedPidRecycled { .. } => Err(FailureReason::Other(
3599 "runner PID was recycled; refusing to signal it".into(),
3600 )),
3601 }
3602 }
3603}
3604
3605pub struct LifecyclePorts {
3606 pub store: Arc<dyn Store>,
3607 pub github: Arc<dyn LifecycleGithub>,
3608 pub packages: Arc<dyn RuntimePackages>,
3609 pub processes: Arc<dyn ExecutionProvider>,
3610 pub clock: Arc<dyn Clock>,
3611 pub demand: Arc<dyn DemandPersistence>,
3612 pub delay: Arc<dyn RetryDelay>,
3613 pub events: Arc<dyn AttemptEventSink>,
3614 pub reconcile_events: Arc<dyn EventSink>,
3615}
3616
3617impl fmt::Debug for LifecyclePorts {
3618 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3619 f.debug_struct("LifecyclePorts")
3620 .field("store", &self.store)
3621 .field("github", &self.github)
3622 .field("packages", &self.packages)
3623 .field("processes", &self.processes)
3624 .finish_non_exhaustive()
3625 }
3626}
3627
3628#[derive(Debug, thiserror::Error)]
3629pub enum LifecycleError {
3630 #[error("attempt journal operation failed")]
3631 Journal,
3632 #[error("attempt {0} is not in the journal")]
3633 Missing(AttemptId),
3634 #[error("attempt lifecycle transition was refused")]
3635 Transition,
3636 #[error("startup recovery has not completed")]
3637 RecoveryIncomplete,
3638 #[error("the persistent slot was not cleaned: {detail}")]
3648 SlotQuarantined {
3649 class: &'static str,
3651 detail: String,
3652 },
3653 #[error("{}", FailureReason::WorkspaceCleanupDeferred)]
3657 WorkspaceCleanupDeferred,
3658 #[error("isolated environment cleanup is deferred")]
3659 EnvironmentCleanupDeferred,
3660 #[error("runner lifecycle failed: {0}")]
3661 Failed(FailureReason),
3662}
3663
3664impl LifecycleError {
3665 fn reason(&self) -> FailureReason {
3666 match self {
3667 Self::Failed(reason) => reason.clone(),
3668 Self::RecoveryIncomplete => FailureReason::Other("startup recovery incomplete".into()),
3669 Self::Journal => FailureReason::Other("attempt journal operation failed".into()),
3670 Self::Missing(_) => FailureReason::Other("attempt disappeared from the journal".into()),
3671 Self::Transition => FailureReason::Other("attempt transition was refused".into()),
3672 Self::SlotQuarantined { .. } => FailureReason::Other(self.to_string()),
3675 Self::WorkspaceCleanupDeferred => FailureReason::WorkspaceCleanupDeferred,
3676 Self::EnvironmentCleanupDeferred => FailureReason::Other(self.to_string()),
3677 }
3678 }
3679}
3680
3681#[derive(Debug)]
3683pub struct LifecycleLauncher {
3684 host_id: HostId,
3685 app_paths: runner_manager_platform::paths::AppPaths,
3686 diagnostics_root: PathBuf,
3687 runner_group_id: u64,
3688 timeouts: RecoveryTimeouts,
3689 retry: RetryPolicy,
3690 cancel: CancelToken,
3691 ports: LifecyclePorts,
3692 recovery_complete: Mutex<bool>,
3693 versions: Mutex<BTreeMap<AttemptId, RunnerVersion>>,
3694 pending_replacements: Mutex<BTreeMap<AttemptId, ReplacementIntent>>,
3695 cleanup_retries: Mutex<CleanBackoff>,
3699 turns: Arc<tokio::sync::Semaphore>,
3700 bringing_up: Mutex<BTreeSet<AttemptId>>,
3703}
3704
3705struct BroughtUpOnDrop<'a> {
3708 attempts: &'a Mutex<BTreeSet<AttemptId>>,
3709 id: AttemptId,
3710}
3711
3712impl Drop for BroughtUpOnDrop<'_> {
3713 fn drop(&mut self) {
3714 self.attempts
3715 .lock()
3716 .unwrap_or_else(std::sync::PoisonError::into_inner)
3717 .remove(&self.id);
3718 }
3719}
3720
3721#[must_use]
3726pub fn preparation_turns() -> Arc<tokio::sync::Semaphore> {
3727 let cores = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
3728 Arc::new(tokio::sync::Semaphore::new((cores / 4).max(2)))
3729}
3730
3731#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3732enum ReconcileProgress {
3733 Reconciled,
3734 Deferred,
3735 Replacement {
3736 attempt: AttemptId,
3737 operation: &'static str,
3738 },
3739}
3740
3741impl LifecycleLauncher {
3742 #[must_use]
3743 pub fn new(
3744 host_id: HostId,
3745 app_paths: runner_manager_platform::paths::AppPaths,
3746 diagnostics_root: impl Into<PathBuf>,
3747 runner_group_id: u64,
3748 timeouts: RecoveryTimeouts,
3749 retry: RetryPolicy,
3750 ports: LifecyclePorts,
3751 ) -> Self {
3752 Self {
3753 host_id,
3754 app_paths,
3755 diagnostics_root: diagnostics_root.into(),
3756 runner_group_id,
3757 timeouts,
3758 retry,
3759 cancel: CancelToken::new(),
3760 ports,
3761 recovery_complete: Mutex::new(false),
3762 versions: Mutex::new(BTreeMap::new()),
3763 pending_replacements: Mutex::new(BTreeMap::new()),
3764 cleanup_retries: Mutex::new(CleanBackoff::default()),
3765 turns: preparation_turns(),
3766 bringing_up: Mutex::default(),
3767 }
3768 }
3769
3770 #[must_use]
3773 pub fn with_preparation_turns(mut self, turns: Arc<tokio::sync::Semaphore>) -> Self {
3774 self.turns = turns;
3775 self
3776 }
3777
3778 pub async fn recover_startup(
3782 &self,
3783 policies: &[ScalePolicy],
3784 ) -> Result<Vec<ReplacementIntent>, LifecycleError> {
3785 *self
3786 .recovery_complete
3787 .lock()
3788 .map_err(|_| LifecycleError::Journal)? = false;
3789 let by_id: BTreeMap<_, _> = policies.iter().map(|policy| (policy.id, policy)).collect();
3790 let attempts = self
3791 .ports
3792 .store
3793 .attempts()
3794 .map_err(|_| LifecycleError::Journal)?;
3795 let owned = self.ports.processes.enumerate_owned(self.host_id);
3796 for identity in &owned {
3797 if let EnvironmentIdentity::Isolated { attempt: id, .. } = identity
3798 && !attempts.iter().any(|attempt| {
3799 attempt.state() != AttemptState::Cleaned
3800 && identity_matches_intent(self.host_id, attempt, identity)
3801 })
3802 {
3803 self.ports
3805 .events
3806 .emit(AttemptEvent::QuarantinedOwnedResource { attempt: *id });
3807 tracing::warn!(host_id = %self.host_id, attempt_id = %id, reason = "owned_isolated_resource_without_matching_journal", "provider resource quarantined for operator review");
3808 }
3809 }
3810 let mut unresolved = false;
3811 for mut attempt in attempts {
3812 let Some(policy) = by_id.get(&attempt.policy_id) else {
3813 if attempt.counts_against_capacity() {
3814 unresolved = true;
3815 }
3816 continue;
3817 };
3818 authorize(self.host_id, policy, &attempt).map_err(|_| LifecycleError::Journal)?;
3819 let allocation_resources = owned
3820 .iter()
3821 .filter(|identity| identity_same_allocation(self.host_id, &attempt, identity))
3822 .collect::<Vec<_>>();
3823 if allocation_resources.len() > 1 {
3824 self.ports
3825 .events
3826 .emit(AttemptEvent::QuarantinedOwnedResource {
3827 attempt: attempt.id,
3828 });
3829 tracing::warn!(attempt_id = %attempt.id, reason = "duplicate_isolated_resource_ownership", "provider resources quarantined for operator review");
3830 }
3831 if attempt.state() != AttemptState::Cleaned
3832 && matches!(
3833 attempt.execution(),
3834 AttemptExecution::Isolated {
3835 environment_id: None,
3836 ..
3837 }
3838 )
3839 && allocation_resources.len() == 1
3840 && let Some(EnvironmentIdentity::Isolated { environment_id, .. }) =
3841 allocation_resources.first().copied()
3842 {
3843 attempt
3844 .prepared_environment(environment_id.clone())
3845 .map_err(|_| LifecycleError::Transition)?;
3846 if attempt.state() == AttemptState::Preparing {
3847 attempt
3848 .mark_prepared(self.ports.clock.now())
3849 .map_err(|_| LifecycleError::Transition)?;
3850 }
3851 self.record(&attempt)?;
3852 }
3853 match self.reconcile_one(policy, attempt).await? {
3854 ReconcileProgress::Deferred => {
3855 if policy.execution_policy().is_native() {
3856 unresolved = true;
3857 }
3858 }
3859 ReconcileProgress::Replacement { attempt, operation } => {
3860 self.pending_replacements
3861 .lock()
3862 .map_err(|_| LifecycleError::Journal)?
3863 .insert(
3864 attempt,
3865 ReplacementIntent {
3866 policy: policy.id,
3867 previous_attempt: attempt,
3868 operation,
3869 },
3870 );
3871 }
3872 ReconcileProgress::Reconciled => {}
3873 }
3874 }
3875 if unresolved {
3876 return Err(LifecycleError::RecoveryIncomplete);
3877 }
3878 *self
3879 .recovery_complete
3880 .lock()
3881 .map_err(|_| LifecycleError::Journal)? = true;
3882 Ok(self
3883 .pending_replacements
3884 .lock()
3885 .map_err(|_| LifecycleError::Journal)?
3886 .values()
3887 .copied()
3888 .collect())
3889 }
3890
3891 pub async fn supervise(
3893 &self,
3894 policy: &ScalePolicy,
3895 ) -> Result<Vec<ReplacementIntent>, LifecycleError> {
3896 let mut replacements = Vec::new();
3897 self.pending_replacements
3898 .lock()
3899 .map_err(|_| LifecycleError::Journal)?
3900 .retain(|_, intent| {
3901 if intent.policy == policy.id {
3902 replacements.push(*intent);
3903 false
3904 } else {
3905 true
3906 }
3907 });
3908 let attempts = {
3913 let bringing_up = self
3914 .bringing_up
3915 .lock()
3916 .map_err(|_| LifecycleError::Journal)?;
3917 let mut attempts = self
3918 .ports
3919 .store
3920 .attempts_for_policy(policy.id)
3921 .map_err(|_| LifecycleError::Journal)?;
3922 attempts.retain(|attempt| !bringing_up.contains(&attempt.id));
3923 attempts
3924 };
3925 for attempt in attempts {
3926 authorize(self.host_id, policy, &attempt).map_err(|_| LifecycleError::Journal)?;
3927 if let ReconcileProgress::Replacement { attempt, operation } =
3928 self.reconcile_one(policy, attempt).await?
3929 {
3930 replacements.push(ReplacementIntent {
3931 policy: policy.id,
3932 previous_attempt: attempt,
3933 operation,
3934 });
3935 }
3936 }
3937 Ok(replacements)
3938 }
3939
3940 async fn reconcile_one(
3941 &self,
3942 policy: &ScalePolicy,
3943 mut attempt: RunnerAttempt,
3944 ) -> Result<ReconcileProgress, LifecycleError> {
3945 if !attempt.execution().is_native() {
3946 return self.reconcile_isolated_one(policy, attempt).await;
3947 }
3948 if attempt.state() == AttemptState::Cleaned {
3949 if matches!(attempt.workspace(), AttemptWorkspace::Ephemeral)
3958 && attempt.runtime_path().exists()
3959 {
3960 let now = self.ports.clock.now();
3968 if !self.cleanup_retries().is_due(attempt.id, now) {
3969 return Ok(ReconcileProgress::Reconciled);
3970 }
3971 match self.scrub_workspace(&attempt) {
3972 Ok(()) => self.cleanup_retries().succeeded(attempt.id),
3973 Err(LifecycleError::WorkspaceCleanupDeferred) => {
3974 self.cleanup_retries().failed(attempt.id, now);
3975 self.ports
3976 .reconcile_events
3977 .emit(LifecycleEvent::AttemptCleanFailed {
3978 policy: attempt.policy_id,
3979 attempt: attempt.id,
3980 reason: "late_ephemeral_workspace_could_not_be_removed",
3981 });
3982 }
3983 Err(error) => return Err(error),
3984 }
3985 }
3986 return Ok(ReconcileProgress::Reconciled);
3987 }
3988 if attempt.is_terminal() {
3989 self.clean_or_quarantine(&mut attempt)?;
3990 return Ok(ReconcileProgress::Reconciled);
3991 }
3992 let process_alive = self
3993 .ports
3994 .processes
3995 .inspect(&attempt)
3996 .map_err(LifecycleError::Failed)?
3997 == EnvironmentState::Running;
3998 let github = self
3999 .ports
4000 .github
4001 .observe(&policy.target, attempt.id, &self.cancel)
4002 .await;
4003
4004 if let Some(runner_id) = github.runner_id
4009 && read_runner_id(attempt.runtime_path()).is_none()
4010 {
4011 write_runner_id(attempt.runtime_path(), runner_id)?;
4012 self.ports
4013 .events
4014 .emit(AttemptEvent::RemoteIdentityRecovered {
4015 attempt: attempt.id,
4016 runner_id,
4017 });
4018 }
4019
4020 if attempt.state() == AttemptState::JitReceived
4025 && process_alive
4026 && let Some(pid) = self
4027 .ports
4028 .processes
4029 .recovered_pid(&attempt)
4030 .map_err(LifecycleError::Failed)?
4031 {
4032 attempt
4033 .started(pid, self.ports.clock.now())
4034 .map_err(|_| LifecycleError::Transition)?;
4035 self.record(&attempt)?;
4036 }
4037
4038 if !process_alive
4051 && github.status != GithubRunnerObservation::Unreachable
4052 && matches!(
4053 attempt.state(),
4054 AttemptState::Starting | AttemptState::Idle | AttemptState::Busy
4055 )
4056 && self.ports.processes.completed_successfully(&attempt)
4057 {
4058 let state = attempt.state();
4059 let outcome = match (state, JobEvidence::of(attempt.runtime_path())) {
4060 (AttemptState::Starting | AttemptState::Idle, JobEvidence::RanAJob) => {
4063 known_runner_id(&attempt, github.runner_id)
4064 .map(|runner_id| (Some(runner_id), AttemptOutcome::CompletedJob))
4065 }
4066 (AttemptState::Busy, JobEvidence::RanAJob) => {
4067 Some((None, AttemptOutcome::CompletedJob))
4068 }
4069 (AttemptState::Busy, JobEvidence::NoJob) => {
4070 Some((None, AttemptOutcome::ExitedIdleWithoutWork))
4071 }
4072 (AttemptState::Busy, JobEvidence::Unknown)
4073 if github.status == GithubRunnerObservation::NotRegistered =>
4074 {
4075 Some((None, AttemptOutcome::CompletedJob))
4076 }
4077 _ => None,
4078 };
4079 if let Some((walk_to_busy, outcome)) = outcome {
4080 if matches!(github.status, GithubRunnerObservation::Registered { .. }) {
4087 self.deregister_runner(policy, &attempt).await;
4088 }
4089 let now = self.ports.clock.now();
4090 match (walk_to_busy, &outcome) {
4091 (Some(runner_id), _) => {
4092 self.transition(&mut attempt, |a| a.assigned_job(runner_id, now))?;
4093 }
4094 (None, AttemptOutcome::ExitedIdleWithoutWork) => {
4095 self.transition(&mut attempt, |a| a.assignment_not_taken(now))?;
4096 }
4097 (None, _) => {}
4098 }
4099 self.conclude(&mut attempt, outcome)?;
4100 self.clean_or_quarantine(&mut attempt)?;
4101 return Ok(ReconcileProgress::Reconciled);
4102 }
4103 }
4104
4105 if process_alive
4110 && attempt.state() == AttemptState::Busy
4111 && github.status == (GithubRunnerObservation::Registered { busy: false })
4112 && JobEvidence::of(attempt.runtime_path()) == JobEvidence::NoJob
4113 {
4114 let now = self.ports.clock.now();
4115 self.transition(&mut attempt, |a| a.assignment_not_taken(now))?;
4116 return Ok(ReconcileProgress::Reconciled);
4117 }
4118
4119 if self.ports.processes.has_terminate_intent(&attempt) && !process_alive {
4122 self.deregister_runner(policy, &attempt).await;
4123 self.conclude(
4124 &mut attempt,
4125 AttemptOutcome::failed(FailureReason::TerminatedAfterRegistrationTimeout),
4126 )?;
4127 self.clean_or_quarantine(&mut attempt)?;
4128 return Ok(ReconcileProgress::Replacement {
4129 attempt: attempt.id,
4130 operation: "registration_timeout_replacement",
4131 });
4132 }
4133
4134 if matches!(
4140 attempt.state(),
4141 AttemptState::Allocated | AttemptState::JitReceived
4142 ) && !process_alive
4143 && matches!(github.status, GithubRunnerObservation::Registered { .. })
4144 {
4145 if attempt.state() == AttemptState::Allocated {
4146 attempt
4147 .jit_received(self.ports.clock.now())
4148 .map_err(|_| LifecycleError::Transition)?;
4149 self.record(&attempt)?;
4150 }
4151 self.deregister_runner(policy, &attempt).await;
4152 self.conclude(
4153 &mut attempt,
4154 AttemptOutcome::failed(FailureReason::JitExpired),
4155 )?;
4156 self.clean_or_quarantine(&mut attempt)?;
4157 return Ok(ReconcileProgress::Replacement {
4158 attempt: attempt.id,
4159 operation: "jit_expired_replacement",
4160 });
4161 }
4162
4163 match recovery_decision(
4164 &attempt,
4165 RecoveryObservation {
4166 process_alive,
4167 github: github.status,
4168 },
4169 self.timeouts,
4170 self.ports.clock.as_ref(),
4171 ) {
4172 RecoveryDecision::Nothing | RecoveryDecision::Wait => Ok(ReconcileProgress::Reconciled),
4173 RecoveryDecision::Defer => Ok(ReconcileProgress::Deferred),
4174 RecoveryDecision::Adopt => {
4175 self.ports.events.emit(AttemptEvent::Adopted {
4176 attempt: attempt.id,
4177 });
4178 Ok(ReconcileProgress::Reconciled)
4179 }
4180 RecoveryDecision::Clean => {
4181 self.clean_or_quarantine(&mut attempt)?;
4182 Ok(ReconcileProgress::Reconciled)
4183 }
4184 RecoveryDecision::Observe(state) => {
4185 let runner_id = known_runner_id(&attempt, github.runner_id)
4186 .ok_or(LifecycleError::Transition)?;
4187 match state {
4188 AttemptState::JitReceived => attempt
4189 .jit_received(self.ports.clock.now())
4190 .map_err(|_| LifecycleError::Transition)?,
4191 AttemptState::Starting => {
4192 let pid = attempt.process_id().ok_or(LifecycleError::Transition)?;
4193 attempt
4194 .started(pid, self.ports.clock.now())
4195 .map_err(|_| LifecycleError::Transition)?;
4196 }
4197 AttemptState::Idle => attempt
4198 .registered_idle(runner_id, self.ports.clock.now())
4199 .map_err(|_| LifecycleError::Transition)?,
4200 AttemptState::Busy => attempt
4201 .assigned_job(runner_id, self.ports.clock.now())
4202 .map_err(|_| LifecycleError::Transition)?,
4203 _ => return Err(LifecycleError::Transition),
4204 }
4205 self.record(&attempt)?;
4206 Ok(ReconcileProgress::Reconciled)
4207 }
4208 RecoveryDecision::Conclude(outcome) => {
4209 let replacement = replacement_operation(&outcome);
4210 if matches!(github.status, GithubRunnerObservation::Registered { .. }) {
4215 self.deregister_runner(policy, &attempt).await;
4216 }
4217 self.conclude(&mut attempt, outcome)?;
4218 self.clean_or_quarantine(&mut attempt)?;
4219 Ok(
4220 replacement.map_or(ReconcileProgress::Reconciled, |operation| {
4221 ReconcileProgress::Replacement {
4222 attempt: attempt.id,
4223 operation,
4224 }
4225 }),
4226 )
4227 }
4228 RecoveryDecision::Terminate(payload) => {
4229 let idle_exit = payload.is_idle_exit();
4239 self.ports
4240 .processes
4241 .record_terminate_intent(&attempt)
4242 .map_err(LifecycleError::Failed)?;
4243 self.ports.events.emit(AttemptEvent::TerminateIntent {
4244 attempt: attempt.id,
4245 });
4246 if self
4247 .ports
4248 .processes
4249 .stop(&attempt)
4250 .map_err(LifecycleError::Failed)?
4251 == ProviderStop::StillRunning
4252 {
4253 return Ok(ReconcileProgress::Deferred);
4254 }
4255 if self
4256 .ports
4257 .processes
4258 .inspect(&attempt)
4259 .map_err(LifecycleError::Failed)?
4260 == EnvironmentState::Running
4261 {
4262 return Ok(ReconcileProgress::Deferred);
4263 }
4264 self.ports.events.emit(AttemptEvent::Terminated {
4265 attempt: attempt.id,
4266 });
4267 let outcome = if idle_exit {
4273 AttemptOutcome::ExitedIdleWithoutWork
4274 } else {
4275 AttemptOutcome::failed(FailureReason::TerminatedAfterRegistrationTimeout)
4276 };
4277 self.deregister_runner(policy, &attempt).await;
4278 self.conclude(&mut attempt, outcome)?;
4279 self.clean_or_quarantine(&mut attempt)?;
4280 if idle_exit {
4284 Ok(ReconcileProgress::Reconciled)
4285 } else {
4286 Ok(ReconcileProgress::Replacement {
4287 attempt: attempt.id,
4288 operation: "registration_timeout_replacement",
4289 })
4290 }
4291 }
4292 }
4293 }
4294
4295 async fn reconcile_isolated_one(
4296 &self,
4297 policy: &ScalePolicy,
4298 mut attempt: RunnerAttempt,
4299 ) -> Result<ReconcileProgress, LifecycleError> {
4300 if attempt.state() == AttemptState::Cleaned {
4301 return Ok(ReconcileProgress::Reconciled);
4302 }
4303 if attempt.is_terminal() {
4304 self.clean_or_quarantine(&mut attempt)?;
4305 return Ok(ReconcileProgress::Reconciled);
4306 }
4307 if matches!(
4308 attempt.state(),
4309 AttemptState::Allocated | AttemptState::Preparing | AttemptState::Prepared
4310 ) {
4311 self.conclude(
4314 &mut attempt,
4315 AttemptOutcome::failed(FailureReason::Other(
4316 "isolated preparation interrupted".into(),
4317 )),
4318 )?;
4319 self.clean_or_quarantine(&mut attempt)?;
4320 return Ok(ReconcileProgress::Reconciled);
4321 }
4322 let Ok(provider_state) = self.ports.processes.recover(&attempt) else {
4323 return Ok(ReconcileProgress::Deferred);
4324 };
4325 if provider_state != EnvironmentState::Missing {
4326 let Ok(owned) = self.ports.processes.owns(&attempt) else {
4327 return Ok(ReconcileProgress::Deferred);
4328 };
4329 if !owned {
4330 self.ports
4331 .events
4332 .emit(AttemptEvent::QuarantinedOwnedResource {
4333 attempt: attempt.id,
4334 });
4335 tracing::warn!(attempt_id = %attempt.id, reason = "isolated_resource_ownership_mismatch", "provider resource quarantined for operator review");
4336 return Ok(ReconcileProgress::Deferred);
4337 }
4338 }
4339 let live = matches!(
4340 provider_state,
4341 EnvironmentState::Starting | EnvironmentState::Running
4342 );
4343 if attempt.state() == AttemptState::JitReceived && live {
4344 attempt
4345 .started_isolated(self.ports.clock.now())
4346 .map_err(|_| LifecycleError::Transition)?;
4347 self.record(&attempt)?;
4348 }
4349 let github = self
4350 .ports
4351 .github
4352 .observe(&policy.target, attempt.id, &self.cancel)
4353 .await;
4354 if let Some(runner_id) = github.runner_id
4355 && read_runner_id(attempt.runtime_path()).is_none()
4356 {
4357 write_runner_id(attempt.runtime_path(), runner_id)?;
4358 }
4359 if attempt.state() == AttemptState::Busy
4363 && !live
4364 && github.status == GithubRunnerObservation::NotRegistered
4365 && self.ports.processes.completed_successfully(&attempt)
4366 {
4367 self.conclude(&mut attempt, AttemptOutcome::CompletedJob)?;
4368 self.clean_or_quarantine(&mut attempt)?;
4369 return Ok(ReconcileProgress::Reconciled);
4370 }
4371 match recovery_decision(
4372 &attempt,
4373 RecoveryObservation {
4374 process_alive: live,
4375 github: github.status,
4376 },
4377 self.timeouts,
4378 self.ports.clock.as_ref(),
4379 ) {
4380 RecoveryDecision::Nothing | RecoveryDecision::Wait => Ok(ReconcileProgress::Reconciled),
4381 RecoveryDecision::Defer => Ok(ReconcileProgress::Deferred),
4382 RecoveryDecision::Adopt => {
4383 self.ports.events.emit(AttemptEvent::Adopted {
4384 attempt: attempt.id,
4385 });
4386 Ok(ReconcileProgress::Reconciled)
4387 }
4388 RecoveryDecision::Clean => {
4389 self.clean_or_quarantine(&mut attempt)?;
4390 Ok(ReconcileProgress::Reconciled)
4391 }
4392 RecoveryDecision::Observe(state) => {
4393 let runner_id = github
4394 .runner_id
4395 .or_else(|| read_runner_id(attempt.runtime_path()))
4396 .ok_or(LifecycleError::Transition)?;
4397 match state {
4398 AttemptState::Idle => attempt
4399 .registered_idle(runner_id, self.ports.clock.now())
4400 .map_err(|_| LifecycleError::Transition)?,
4401 AttemptState::Busy => attempt
4402 .assigned_job(runner_id, self.ports.clock.now())
4403 .map_err(|_| LifecycleError::Transition)?,
4404 _ => return Err(LifecycleError::Transition),
4405 }
4406 self.record(&attempt)?;
4407 Ok(ReconcileProgress::Reconciled)
4408 }
4409 RecoveryDecision::Conclude(outcome) | RecoveryDecision::Terminate(outcome) => {
4410 let replacement = replacement_operation(&outcome);
4411 if matches!(github.status, GithubRunnerObservation::Registered { .. }) {
4412 self.deregister_runner(policy, &attempt).await;
4413 }
4414 self.conclude(&mut attempt, outcome)?;
4415 self.clean_or_quarantine(&mut attempt)?;
4416 Ok(
4417 replacement.map_or(ReconcileProgress::Reconciled, |operation| {
4418 ReconcileProgress::Replacement {
4419 attempt: attempt.id,
4420 operation,
4421 }
4422 }),
4423 )
4424 }
4425 }
4426 }
4427
4428 fn transition(
4430 &self,
4431 attempt: &mut RunnerAttempt,
4432 edge: impl FnOnce(
4433 &mut RunnerAttempt,
4434 ) -> Result<(), runner_manager_domain::attempt::AttemptError>,
4435 ) -> Result<(), LifecycleError> {
4436 edge(attempt).map_err(|_| LifecycleError::Transition)?;
4437 self.record(attempt)
4438 }
4439
4440 fn record(&self, attempt: &RunnerAttempt) -> Result<(), LifecycleError> {
4441 self.ports
4442 .store
4443 .record_attempt(attempt)
4444 .map_err(|_| LifecycleError::Journal)?;
4445 self.ports.events.emit(AttemptEvent::State {
4446 attempt: attempt.id,
4447 state: attempt.state(),
4448 });
4449 Ok(())
4450 }
4451
4452 async fn deregister_runner(&self, policy: &ScalePolicy, attempt: &RunnerAttempt) {
4473 let Some(runner_id) = attempt
4474 .github_runner_id()
4475 .or_else(|| read_runner_id(attempt.runtime_path()))
4476 else {
4477 return;
4478 };
4479 if self
4480 .ports
4481 .github
4482 .deregister(&policy.target, runner_id, &self.cancel)
4483 .await
4484 {
4485 self.ports.events.emit(AttemptEvent::Deregistered {
4486 attempt: attempt.id,
4487 runner_id,
4488 });
4489 } else {
4490 tracing::warn!(
4491 attempt = %attempt.id,
4492 runner_id,
4493 "the runner registration could not be removed from GitHub; it will show in the \
4494 target's runner settings until GitHub retires it or a later pass removes it"
4495 );
4496 }
4497 }
4498
4499 fn conclude(
4500 &self,
4501 attempt: &mut RunnerAttempt,
4502 outcome: AttemptOutcome,
4503 ) -> Result<(), LifecycleError> {
4504 attempt
4505 .conclude(outcome.clone(), self.ports.clock.now())
4506 .map_err(|_| LifecycleError::Transition)?;
4507 self.record(attempt)?;
4508 self.ports.events.emit(AttemptEvent::Concluded {
4509 attempt: attempt.id,
4510 outcome: OutcomeKind::of(&outcome),
4511 });
4512 Ok(())
4513 }
4514
4515 fn clean_or_quarantine(&self, attempt: &mut RunnerAttempt) -> Result<(), LifecycleError> {
4535 let now = self.ports.clock.now();
4536 if !self.cleanup_retries().is_due(attempt.id, now) {
4537 return Ok(());
4538 }
4539 let reason = match self.clean_attempt(attempt) {
4540 Err(LifecycleError::SlotQuarantined { class, .. }) => class,
4541 Err(LifecycleError::WorkspaceCleanupDeferred) => {
4542 failure_reason_kind(&FailureReason::WorkspaceCleanupDeferred)
4543 }
4544 Err(LifecycleError::EnvironmentCleanupDeferred) => {
4545 "isolated_environment_cleanup_deferred"
4546 }
4547 other => {
4548 if other.is_ok() {
4549 self.cleanup_retries().succeeded(attempt.id);
4550 }
4551 return other;
4552 }
4553 };
4554 self.cleanup_retries().failed(attempt.id, now);
4555 self.ports
4556 .reconcile_events
4557 .emit(LifecycleEvent::AttemptCleanFailed {
4558 policy: attempt.policy_id,
4559 attempt: attempt.id,
4560 reason,
4561 });
4562 Ok(())
4563 }
4564
4565 fn cleanup_retries(&self) -> std::sync::MutexGuard<'_, CleanBackoff> {
4566 self.cleanup_retries
4567 .lock()
4568 .unwrap_or_else(std::sync::PoisonError::into_inner)
4569 }
4570
4571 fn clean_attempt(&self, attempt: &mut RunnerAttempt) -> Result<(), LifecycleError> {
4572 if !attempt.execution().is_native() {
4573 return self.clean_isolated_attempt(attempt);
4574 }
4575 match self
4576 .ports
4577 .processes
4578 .destroy(attempt)
4579 .map_err(LifecycleError::Failed)?
4580 {
4581 ProviderDestroy::Destroyed => {}
4582 ProviderDestroy::Deferred(reason) => {
4583 return Err(LifecycleError::Failed(FailureReason::Other(reason.into())));
4584 }
4585 }
4586 let outcome = attempt
4587 .outcome()
4588 .cloned()
4589 .ok_or(LifecycleError::Transition)?;
4590 self.preserve_diagnostics(attempt, &outcome)?;
4591 self.scrub_workspace(attempt)?;
4592 self.ports
4593 .packages
4594 .release(attempt.id)
4595 .map_err(LifecycleError::Failed)?;
4596 attempt
4597 .clean(self.ports.clock.now())
4598 .map_err(|_| LifecycleError::Transition)?;
4599 self.record(attempt)?;
4600 let kind = OutcomeKind::of(&outcome);
4601 self.ports.events.emit(AttemptEvent::Cleaned {
4602 attempt: attempt.id,
4603 outcome: kind,
4604 });
4605 self.ports
4606 .reconcile_events
4607 .emit(LifecycleEvent::AttemptCleaned {
4608 policy: attempt.policy_id,
4609 attempt: attempt.id,
4610 outcome: kind,
4611 });
4612 Ok(())
4613 }
4614
4615 fn clean_isolated_attempt(&self, attempt: &mut RunnerAttempt) -> Result<(), LifecycleError> {
4616 if attempt.state() != AttemptState::Destroying {
4617 attempt
4618 .begin_destroy(self.ports.clock.now())
4619 .map_err(|_| LifecycleError::Transition)?;
4620 self.record(attempt)?;
4621 }
4622 let provider = self.ports.processes.as_ref();
4623 let allocation_resources = provider.enumerate_owned(self.host_id);
4624 let matching_allocation = allocation_resources
4625 .iter()
4626 .filter(|identity| identity_same_allocation(self.host_id, attempt, identity))
4627 .collect::<Vec<_>>();
4628 if matching_allocation.len() > 1
4629 || matching_allocation
4630 .iter()
4631 .any(|identity| !identity_matches_intent(self.host_id, attempt, identity))
4632 {
4633 self.ports
4634 .events
4635 .emit(AttemptEvent::QuarantinedOwnedResource {
4636 attempt: attempt.id,
4637 });
4638 tracing::warn!(attempt_id = %attempt.id, reason = "duplicate_or_mismatched_isolated_ownership", "provider resources quarantined for operator review");
4639 return Err(self.defer_environment_cleanup(attempt)?);
4640 }
4641 let has_identity = matches!(
4642 attempt.execution(),
4643 AttemptExecution::Isolated {
4644 environment_id: Some(_),
4645 ..
4646 }
4647 );
4648 if !has_identity
4649 && allocation_resources
4650 .iter()
4651 .any(|identity| identity_matches_intent(self.host_id, attempt, identity))
4652 {
4653 return Err(self.defer_environment_cleanup(attempt)?);
4654 }
4655 let current = if has_identity {
4656 provider.inspect(attempt)
4657 } else {
4658 provider.recover(attempt)
4661 };
4662 let current = match current {
4663 Ok(state) => state,
4664 Err(_) => return Err(self.defer_environment_cleanup(attempt)?),
4665 };
4666 if !has_identity && current != EnvironmentState::Missing {
4667 return Err(self.defer_environment_cleanup(attempt)?);
4668 }
4669 if current != EnvironmentState::Missing {
4670 if !matches!(provider.owns(attempt), Ok(true)) {
4671 self.ports
4672 .events
4673 .emit(AttemptEvent::QuarantinedOwnedResource {
4674 attempt: attempt.id,
4675 });
4676 tracing::warn!(attempt_id = %attempt.id, reason = "isolated_resource_ownership_mismatch", "provider resource quarantined for operator review");
4677 return Err(self.defer_environment_cleanup(attempt)?);
4678 }
4679 if matches!(
4680 current,
4681 EnvironmentState::Starting | EnvironmentState::Running
4682 ) && !matches!(provider.stop(attempt), Ok(ProviderStop::Stopped))
4683 {
4684 return Err(self.defer_environment_cleanup(attempt)?);
4685 }
4686 if !matches!(provider.destroy(attempt), Ok(ProviderDestroy::Destroyed)) {
4687 return Err(self.defer_environment_cleanup(attempt)?);
4688 }
4689 if !matches!(provider.inspect(attempt), Ok(EnvironmentState::Missing)) {
4690 return Err(self.defer_environment_cleanup(attempt)?);
4691 }
4692 }
4693 if provider
4694 .enumerate_owned(self.host_id)
4695 .iter()
4696 .any(|identity| identity_same_allocation(self.host_id, attempt, identity))
4697 {
4698 return Err(self.defer_environment_cleanup(attempt)?);
4699 }
4700 let outcome = attempt
4701 .outcome()
4702 .cloned()
4703 .ok_or(LifecycleError::Transition)?;
4704 for diagnostic in provider.diagnostics(attempt).into_iter().take(32) {
4707 tracing::debug!(attempt_id = %attempt.id, provider_diagnostic = ?diagnostic, "provider lifecycle diagnostic");
4708 }
4709 self.preserve_diagnostics(attempt, &outcome)?;
4710 self.scrub_workspace(attempt)?;
4711 self.ports
4712 .packages
4713 .release(attempt.id)
4714 .map_err(LifecycleError::Failed)?;
4715 attempt
4716 .clean_isolated(self.ports.clock.now())
4717 .map_err(|_| LifecycleError::Transition)?;
4718 self.record(attempt)?;
4719 self.ports.events.emit(AttemptEvent::Cleaned {
4720 attempt: attempt.id,
4721 outcome: OutcomeKind::of(&outcome),
4722 });
4723 self.ports
4724 .reconcile_events
4725 .emit(LifecycleEvent::AttemptCleaned {
4726 policy: attempt.policy_id,
4727 attempt: attempt.id,
4728 outcome: OutcomeKind::of(&outcome),
4729 });
4730 Ok(())
4731 }
4732
4733 fn defer_environment_cleanup(
4734 &self,
4735 attempt: &mut RunnerAttempt,
4736 ) -> Result<LifecycleError, LifecycleError> {
4737 attempt
4738 .defer_cleanup(self.ports.clock.now())
4739 .map_err(|_| LifecycleError::Transition)?;
4740 self.record(attempt)?;
4741 Ok(LifecycleError::EnvironmentCleanupDeferred)
4742 }
4743
4744 fn scrub_workspace(&self, attempt: &RunnerAttempt) -> Result<(), LifecycleError> {
4754 #[cfg(test)]
4755 {
4756 if matches!(
4760 std::env::var("RUNNER_MANAGER_TEST_MUTANT").as_deref(),
4761 Ok("skip_workspace_cleanup" | "reuse_job_workspace")
4762 ) {
4763 return Ok(());
4764 }
4765 }
4766 match attempt.workspace() {
4767 AttemptWorkspace::Ephemeral => match remove_runtime_tree(attempt.runtime_path()) {
4768 Ok(()) => Ok(()),
4769 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
4770 Err(_) => Err(LifecycleError::WorkspaceCleanupDeferred),
4771 },
4772 AttemptWorkspace::PersistentSlot { slot } => self.scrub_persistent_slot(attempt, slot),
4773 }
4774 }
4775
4776 fn scrub_persistent_slot(
4789 &self,
4790 attempt: &RunnerAttempt,
4791 slot: NonZeroU16,
4792 ) -> Result<(), LifecycleError> {
4793 let configured = self
4798 .ports
4799 .store
4800 .policy(attempt.policy_id)
4801 .map_err(|_| LifecycleError::Journal)?
4802 .and_then(|policy| match policy.workspace_policy() {
4803 WorkspacePolicy::Persistent { root } => Some(root.clone()),
4804 WorkspacePolicy::Ephemeral => None,
4805 });
4806 let runtime = attempt.runtime_path();
4807 self.quarantine_on_refusal(
4808 attempt,
4809 verify_journalled_slot(runtime, slot, configured.as_ref())
4810 .and_then(|()| slot_is_present(runtime))
4811 .and_then(|present| {
4812 if present {
4813 scrub_slot_entries(runtime).and_then(|()| verify_slot_scrubbed(runtime))
4814 } else {
4815 Ok(())
4816 }
4817 }),
4818 )
4819 }
4820
4821 fn quarantine_on_refusal(
4830 &self,
4831 attempt: &RunnerAttempt,
4832 outcome: Result<(), SlotQuarantine>,
4833 ) -> Result<(), LifecycleError> {
4834 let Err(quarantine) = outcome else {
4835 return Ok(());
4836 };
4837 let detail = quarantine.to_string();
4838 tracing::warn!(
4839 attempt = %attempt.id,
4840 policy = %attempt.policy_id,
4841 slot = attempt.workspace().slot_number(),
4842 refusal = quarantine.refusal.class(),
4843 "{detail}"
4844 );
4845 Err(LifecycleError::SlotQuarantined {
4846 class: quarantine.refusal.class(),
4847 detail,
4848 })
4849 }
4850
4851 fn preserve_diagnostics(
4852 &self,
4853 attempt: &RunnerAttempt,
4854 outcome: &AttemptOutcome,
4855 ) -> Result<(), LifecycleError> {
4856 fs::create_dir_all(&self.diagnostics_root).map_err(|_| {
4857 LifecycleError::Failed(FailureReason::Other(
4858 "diagnostics directory could not be created".into(),
4859 ))
4860 })?;
4861 let reason_kind = match outcome {
4864 AttemptOutcome::Failed { reason } => failure_reason_kind(reason),
4865 _ => "none",
4866 };
4867 let diagnostic = format!(
4868 "attempt_id={}\npolicy_id={}\noutcome={}\nreason={}\n",
4869 attempt.id,
4870 attempt.policy_id,
4871 OutcomeKind::of(outcome).as_str(),
4872 reason_kind,
4873 );
4874 fs::write(
4875 self.diagnostics_root.join(format!("{}.log", attempt.id)),
4876 diagnostic,
4877 )
4878 .map_err(|_| {
4879 LifecycleError::Failed(FailureReason::Other(
4880 "redacted diagnostics could not be preserved".into(),
4881 ))
4882 })
4883 }
4884
4885 async fn materialize_with_retry(
4886 &self,
4887 policy: &ScalePolicy,
4888 attempt: &RunnerAttempt,
4889 timings: &mut PackageTimings,
4890 ) -> Result<RunnerVersion, FailureReason> {
4891 let mut issued = 0_u32;
4892 loop {
4893 issued = issued.saturating_add(1);
4894 match self.ports.packages.materialize(attempt, timings).await {
4895 Ok(version) => return Ok(version),
4896 Err(reason)
4897 if package_failure_is_terminal(&reason)
4898 || issued >= self.retry.max_attempts.max(1) =>
4899 {
4900 return Err(reason);
4901 }
4902 Err(reason) => {
4903 if !self.ports.demand.persists(policy.id).await {
4904 return Err(reason);
4905 }
4906 let delay = self.retry.delay(issued);
4907 self.ports.events.emit(AttemptEvent::Retry {
4908 attempt: attempt.id,
4909 operation: "package_materialization",
4910 delay,
4911 });
4912 self.ports.delay.wait(delay).await;
4913 if !self.ports.demand.persists(policy.id).await {
4914 return Err(reason);
4915 }
4916 }
4917 }
4918 }
4919 }
4920
4921 async fn register_with_retry(
4922 &self,
4923 policy: &ScalePolicy,
4924 attempt: AttemptId,
4925 request: &JitRunnerRequest,
4926 ) -> Result<JitRegistration, LifecycleError> {
4927 let mut issued = 0_u32;
4928 loop {
4929 issued = issued.saturating_add(1);
4930 match self
4931 .ports
4932 .github
4933 .register(&policy.target, request, &self.cancel)
4934 .await
4935 {
4936 Ok(registration) => return Ok(registration),
4937 Err(error) if error.terminal => {
4938 return Err(LifecycleError::Failed(error.reason));
4939 }
4940 Err(error) => {
4941 if issued >= self.retry.max_attempts.max(1)
4942 || !self.ports.demand.persists(policy.id).await
4943 {
4944 return Err(LifecycleError::Failed(error.reason));
4945 }
4946 let delay = error
4947 .retry_after
4948 .unwrap_or_else(|| self.retry.delay(issued));
4949 self.ports.events.emit(AttemptEvent::Retry {
4950 attempt,
4951 operation: "jit_request",
4952 delay,
4953 });
4954 self.ports.delay.wait(delay).await;
4955 if !self.ports.demand.persists(policy.id).await {
4956 return Err(LifecycleError::Failed(error.reason));
4957 }
4958 }
4959 }
4960 }
4961 }
4962
4963 fn allocate_workspace(
4973 &self,
4974 policy: &ScalePolicy,
4975 id: AttemptId,
4976 ) -> Result<Placement, LifecycleError> {
4977 let placement = match policy.workspace_policy() {
4978 WorkspacePolicy::Persistent { root } => self.allocate_persistent_slot(policy, root),
4983 WorkspacePolicy::Ephemeral => self.allocate_disposable(policy, id),
4984 };
4985 if placement.is_ok() {
4988 self.root_accepted(policy.id);
4989 }
4990 placement
4991 }
4992
4993 fn configured_host_root(&self) -> Result<Option<LocalAbsolutePath>, LifecycleError> {
5000 let host = self
5001 .ports
5002 .store
5003 .host(self.host_id)
5004 .map_err(|_| LifecycleError::Journal)?
5005 .ok_or_else(|| LifecycleError::Failed(FailureReason::Other("host not found".into())))?;
5006 Ok(host.runner_root_override.clone())
5007 }
5008
5009 fn resolvable_host_root(&self) -> Result<Option<LocalAbsolutePath>, LifecycleError> {
5013 Ok(self
5014 .configured_host_root()?
5015 .or_else(|| default_runner_root(&self.app_paths).ok()))
5016 }
5017
5018 fn effective_host_root(
5023 &self,
5024 policy: &ScalePolicy,
5025 ) -> Result<LocalAbsolutePath, LifecycleError> {
5026 match self.configured_host_root()? {
5027 Some(configured) => Ok(configured),
5028 None => default_runner_root(&self.app_paths).map_err(|error| {
5029 self.root_refused(policy.id, "the platform default runner root", error)
5032 }),
5033 }
5034 }
5035
5036 fn root_refused(&self, policy: PolicyId, root: &str, error: RunnerRootError) -> LifecycleError {
5049 let _ = runner_manager_platform::service::record_runner_root_refusal(
5050 &self.app_paths,
5051 &policy.to_string(),
5052 self.ports.clock.now(),
5053 error.kind(),
5054 root,
5055 &error.to_string(),
5056 );
5057 root_failure(error)
5058 }
5059
5060 fn root_accepted(&self, policy: PolicyId) {
5067 let _ = runner_manager_platform::service::clear_runner_root_refusal(
5068 &self.app_paths,
5069 &policy.to_string(),
5070 );
5071 }
5072
5073 fn allocate_disposable(
5076 &self,
5077 policy: &ScalePolicy,
5078 id: AttemptId,
5079 ) -> Result<Placement, LifecycleError> {
5080 let effective_root = self.effective_host_root(policy)?;
5081 RootPreflight::new(&self.app_paths)
5082 .check(&RootOwner::Host, &effective_root)
5083 .map_err(|error| self.root_refused(policy.id, effective_root.as_str(), error))?;
5084 let runtime = effective_root.as_path().join({
5085 #[cfg(test)]
5086 {
5087 if std::env::var("RUNNER_MANAGER_TEST_MUTANT").as_deref()
5088 == Ok("reuse_job_workspace")
5089 {
5090 "mutant-shared-workspace".to_owned()
5091 } else {
5092 workspace_name(id)
5093 }
5094 }
5095 #[cfg(not(test))]
5096 {
5097 workspace_name(id)
5098 }
5099 });
5100 fs::create_dir_all(&runtime)
5101 .map_err(|_| LifecycleError::Failed(FailureReason::ProcessStartFailed))?;
5102 Ok(Placement {
5103 runtime,
5104 workspace: AttemptWorkspace::Ephemeral,
5105 })
5106 }
5107
5108 fn allocate_persistent_slot(
5120 &self,
5121 policy: &ScalePolicy,
5122 root: &LocalAbsolutePath,
5123 ) -> Result<Placement, LifecycleError> {
5124 let ceiling = policy.max_capacity().ok_or_else(|| {
5127 LifecycleError::Failed(FailureReason::Other(
5128 "a persistent workspace needs the policy's max_capacity to bound its slots"
5129 .to_string(),
5130 ))
5131 })?;
5132 let leases = self
5133 .ports
5134 .store
5135 .slot_leases_for_policy(policy.id)
5136 .map_err(|_| LifecycleError::Journal)?;
5137 let slot = lowest_free_slot(&leases, ceiling).ok_or_else(|| {
5138 LifecycleError::Failed(FailureReason::Other(format!(
5139 "every persistent slot s1 to s{ceiling} for {} is leased by an attempt that has \
5140 not been cleaned, so no slot is free; raise the repository's max capacity, or \
5141 finish cleaning a concluded attempt",
5142 policy.target
5143 )))
5144 })?;
5145 let workspace = AttemptWorkspace::persistent_slot(slot);
5146 let name = workspace
5147 .slot_directory_name()
5148 .expect("a persistent allocation names its slot directory");
5149
5150 let host_root = self.resolvable_host_root()?;
5162 let mut preflight = RootPreflight::new(&self.app_paths);
5163 if let Some(host_root) = host_root {
5164 preflight = preflight.against(RootOwner::Host, host_root);
5165 }
5166 let checked = preflight
5167 .check(&RootOwner::Repository(policy.target.to_string()), root)
5168 .map_err(|error| self.root_refused(policy.id, root.as_str(), error))?;
5169 if let Some(leaf) = checked.leaf_to_create() {
5170 fs::create_dir(leaf).map_err(|source| {
5171 LifecycleError::Failed(FailureReason::Other(format!(
5172 "the persistent workspace root {} could not be created: {source}",
5173 leaf.display()
5174 )))
5175 })?;
5176 }
5177
5178 let slot_path = runner_root::derive_child(root, &name)
5182 .map_err(|error| self.root_refused(policy.id, root.as_str(), error))?;
5183 create_or_validate_slot(slot_path.as_path())?;
5184 runner_root::verify_containment(root, &slot_path)
5185 .map_err(|error| self.root_refused(policy.id, root.as_str(), error))?;
5186 accept_reusable_slot(slot_path.as_path())?;
5187 Ok(Placement {
5188 runtime: slot_path.as_path().to_path_buf(),
5189 workspace,
5190 })
5191 }
5192
5193 fn record_allocation(&self, attempt: &RunnerAttempt) -> Result<(), LifecycleError> {
5205 match self.ports.store.record_attempt(attempt) {
5206 Ok(()) => {
5207 self.ports.events.emit(AttemptEvent::State {
5208 attempt: attempt.id,
5209 state: attempt.state(),
5210 });
5211 Ok(())
5212 }
5213 Err(error @ StoreError::SlotAlreadyLeased { .. }) => Err(LifecycleError::Failed(
5214 FailureReason::Other(error.to_string()),
5215 )),
5216 Err(_) => Err(LifecycleError::Journal),
5217 }
5218 }
5219
5220 fn reserve_attempt(
5223 &self,
5224 policy: &ScalePolicy,
5225 allocation_guard: &AllocationGuard,
5226 ) -> Result<RunnerAttempt, LifecycleError> {
5227 if let Err(error) = self.prune_under_allocation_lock(allocation_guard) {
5230 tracing::warn!(
5231 event = "package_prune_failed",
5232 reason = failure_reason_kind(&error.reason()),
5233 );
5234 }
5235 self.allocate_attempt(policy)
5236 }
5237
5238 async fn finish_attempt(
5239 &self,
5240 policy: &ScalePolicy,
5241 attempt: RunnerAttempt,
5242 waits: ReservationWaits,
5243 ) -> Result<RunnerAttempt, LifecycleError> {
5244 let id = attempt.id;
5245 let _bringing_up = BroughtUpOnDrop {
5246 attempts: &self.bringing_up,
5247 id,
5248 };
5249 let mut stopwatch = LaunchStopwatch::new(waits);
5250 let result = self.launch_steps(policy, attempt, &mut stopwatch).await;
5251 stopwatch.finish();
5252 self.ports.events.emit(AttemptEvent::Launched {
5253 attempt: id,
5254 started: result.is_ok(),
5255 timings: stopwatch.timings,
5256 });
5257 result
5258 }
5259
5260 fn allocate_attempt(&self, policy: &ScalePolicy) -> Result<RunnerAttempt, LifecycleError> {
5263 if !*self
5264 .recovery_complete
5265 .lock()
5266 .map_err(|_| LifecycleError::Journal)?
5267 {
5268 return Err(LifecycleError::RecoveryIncomplete);
5269 }
5270 let capability = self.ports.processes.probe(policy);
5271 if capability != ProviderCapability::Ready {
5272 return Err(LifecycleError::Failed(capability.refusal()));
5273 }
5274 let resolved = self.ports.processes.resolve(policy).map_err(|reason| {
5275 LifecycleError::Failed(if policy.execution_policy().is_native() {
5276 reason
5277 } else {
5278 safe_isolation_reason(reason, IsolationProviderFailure::RuntimeOperationFailed)
5279 })
5280 })?;
5281 if policy.execution_policy().is_native() != resolved.is_none() {
5282 return Err(LifecycleError::Failed(FailureReason::Other(
5283 "provider resolution mismatch".into(),
5284 )));
5285 }
5286 if let Some(resolution) = &resolved {
5287 if resolution.provider_kind == Backend::Auto {
5288 return Err(LifecycleError::Failed(FailureReason::Other(
5289 "provider resolution mismatch".into(),
5290 )));
5291 }
5292 AttemptExecution::Isolated {
5293 provider_kind: resolution.provider_kind,
5294 environment_id: None,
5295 resolved_image: resolution.image.clone(),
5296 generation: "generation-check".into(),
5297 }
5298 .validate()
5299 .map_err(|_| {
5300 LifecycleError::Failed(FailureReason::Other(
5301 "resolved image is incompatible".into(),
5302 ))
5303 })?;
5304 }
5305 policy
5306 .routing_labels()
5307 .ok_or(LifecycleError::Failed(FailureReason::JitRequestFailed))?;
5308 let id = AttemptId::new_random();
5309 let placement = self.allocate_workspace(policy, id)?;
5310 let mut attempt = RunnerAttempt::allocate_in(
5311 id,
5312 policy.id,
5313 placement.runtime,
5314 placement.workspace,
5315 self.ports.clock.now(),
5316 );
5317 if let Some(resolution) = &resolved {
5318 attempt
5319 .allocate_execution(AttemptExecution::Isolated {
5320 provider_kind: resolution.provider_kind,
5321 environment_id: None,
5322 resolved_image: resolution.image.clone(),
5323 generation: uuid::Uuid::new_v4().to_string(),
5324 })
5325 .map_err(|_| LifecycleError::Transition)?;
5326 }
5327 self.bringing_up
5332 .lock()
5333 .map_err(|_| LifecycleError::Journal)?
5334 .insert(id);
5335 if let Err(error) = self.record_allocation(&attempt) {
5336 self.bringing_up
5337 .lock()
5338 .map_err(|_| LifecycleError::Journal)?
5339 .remove(&id);
5340 return Err(error);
5341 }
5342 Ok(attempt)
5343 }
5344
5345 async fn launch_steps(
5346 &self,
5347 policy: &ScalePolicy,
5348 mut attempt: RunnerAttempt,
5349 stopwatch: &mut LaunchStopwatch,
5350 ) -> Result<RunnerAttempt, LifecycleError> {
5351 let id = attempt.id;
5352 let isolated = !attempt.execution().is_native();
5353 let Some(labels) = policy.routing_labels() else {
5354 return self.fail_launch(&mut attempt, FailureReason::JitRequestFailed);
5355 };
5356 stopwatch.begin(LaunchStep::Turn);
5357 let turn = self.turns.acquire().await.ok();
5359 stopwatch.begin(LaunchStep::Materialize);
5360 let package_timings = &mut stopwatch.timings.package;
5361 let version = match self
5362 .materialize_with_retry(policy, &attempt, package_timings)
5363 .await
5364 {
5365 Ok(version) => version,
5366 Err(reason) => return self.fail_launch(&mut attempt, reason),
5367 };
5368 self.versions
5369 .lock()
5370 .map_err(|_| LifecycleError::Journal)?
5371 .insert(id, version);
5372
5373 if isolated {
5374 attempt
5375 .begin_prepare(self.ports.clock.now())
5376 .map_err(|_| LifecycleError::Transition)?;
5377 self.record(&attempt)?;
5378 }
5379 stopwatch.begin(LaunchStep::Prepare);
5380 let preparing = {
5383 let (processes, attempt, policy) = (
5384 Arc::clone(&self.ports.processes),
5385 attempt.clone(),
5386 policy.clone(),
5387 );
5388 tokio::task::spawn_blocking(move || processes.prepare(&attempt, &policy)).await
5389 };
5390 drop(turn);
5391 let prepared = match preparing.unwrap_or(Err(FailureReason::ProcessStartFailed)) {
5392 Ok(prepared) => prepared,
5393 Err(reason) => {
5394 let safe_reason = if isolated {
5395 safe_isolation_reason(reason, IsolationProviderFailure::RuntimeOperationFailed)
5396 } else {
5397 reason
5398 };
5399 return self.fail_launch(&mut attempt, safe_reason);
5400 }
5401 };
5402 if isolated {
5403 let Some(identity) = prepared.identity() else {
5404 return Err(LifecycleError::Failed(FailureReason::IsolationProvider(
5405 IsolationProviderFailure::OwnershipMismatch,
5406 )));
5407 };
5408 if !identity_matches_intent(self.host_id, &attempt, identity) {
5409 return Err(LifecycleError::Failed(FailureReason::IsolationProvider(
5410 IsolationProviderFailure::OwnershipMismatch,
5411 )));
5412 }
5413 let EnvironmentIdentity::Isolated { environment_id, .. } = identity else {
5414 unreachable!()
5415 };
5416 attempt
5417 .prepared_environment(environment_id.clone())
5418 .map_err(|_| LifecycleError::Transition)?;
5419 attempt
5420 .mark_prepared(self.ports.clock.now())
5421 .map_err(|_| LifecycleError::Transition)?;
5422 self.record(&attempt)?;
5423 }
5424
5425 stopwatch.begin(LaunchStep::Register);
5426 let jit_request =
5427 JitRunnerRequest::for_policy(runner_name(id), self.runner_group_id, labels);
5428 let registration = match self.register_with_retry(policy, id, &jit_request).await {
5429 Ok(registration) => registration,
5430 Err(error) => return self.fail_launch(&mut attempt, error.reason()),
5431 };
5432 stopwatch.begin(LaunchStep::Spawn);
5433 let runner_id = registration.runner().id;
5434 write_runner_id(attempt.runtime_path(), runner_id)?;
5435 attempt
5436 .jit_received(self.ports.clock.now())
5437 .map_err(|_| LifecycleError::Transition)?;
5438 self.record(&attempt)?;
5439 let config = registration.into_config();
5440 if isolated {
5441 let started = self
5442 .ports
5443 .processes
5444 .start(prepared, &attempt, OneTimeJitHandoff::new(&config))
5445 .map_err(|failure| {
5446 LifecycleError::Failed(safe_isolation_reason(
5447 failure.reason,
5448 IsolationProviderFailure::RuntimeOperationFailed,
5449 ))
5450 })?;
5451 if !identity_matches_intent(self.host_id, &attempt, &started) {
5452 return Err(LifecycleError::Failed(FailureReason::IsolationProvider(
5453 IsolationProviderFailure::OwnershipMismatch,
5454 )));
5455 }
5456 attempt
5457 .started_isolated(self.ports.clock.now())
5458 .map_err(|_| LifecycleError::Transition)?;
5459 self.record(&attempt)?;
5460 return Ok(attempt);
5461 }
5462 let mut issued = 0_u32;
5463 let pid = loop {
5464 issued = issued.saturating_add(1);
5465 match self.ports.processes.start(
5466 prepared.clone(),
5467 &attempt,
5468 OneTimeJitHandoff::new(&config),
5469 ) {
5470 Ok(EnvironmentIdentity::NativeProcess(pid)) => break pid,
5471 Ok(EnvironmentIdentity::Isolated { .. }) => {
5472 return Err(LifecycleError::Failed(FailureReason::Other(
5476 "unexpected isolated environment identity".into(),
5477 )));
5478 }
5479 Err(error) => {
5480 if let Some(pid) = error.live_pid {
5481 attempt
5482 .started(pid, self.ports.clock.now())
5483 .map_err(|_| LifecycleError::Transition)?;
5484 self.record(&attempt)?;
5485 return Err(LifecycleError::Failed(error.reason));
5486 }
5487 if !error.retryable
5488 || issued >= self.retry.max_attempts.max(1)
5489 || !self.ports.demand.persists(policy.id).await
5490 {
5491 return self.fail_launch(&mut attempt, error.reason);
5492 }
5493 let delay = self.retry.delay(issued);
5494 self.ports.events.emit(AttemptEvent::Retry {
5495 attempt: attempt.id,
5496 operation: "process_start",
5497 delay,
5498 });
5499 self.ports.delay.wait(delay).await;
5500 if !self.ports.demand.persists(policy.id).await {
5501 return self.fail_launch(&mut attempt, error.reason);
5502 }
5503 }
5504 }
5505 };
5506 attempt
5507 .started(pid, self.ports.clock.now())
5508 .map_err(|_| LifecycleError::Transition)?;
5509 self.record(&attempt)?;
5510 Ok(attempt)
5511 }
5512
5513 fn fail_launch<T>(
5514 &self,
5515 attempt: &mut RunnerAttempt,
5516 reason: FailureReason,
5517 ) -> Result<T, LifecycleError> {
5518 self.conclude(attempt, AttemptOutcome::failed(reason.clone()))?;
5519 Err(LifecycleError::Failed(reason))
5520 }
5521
5522 fn prune_under_allocation_lock(&self, guard: &AllocationGuard) -> Result<(), LifecycleError> {
5525 let attempts = self
5526 .ports
5527 .store
5528 .attempts()
5529 .map_err(|_| LifecycleError::Journal)?;
5530 self.ports
5531 .packages
5532 .prune_obsolete_guarded(PruneAuthority::from_launch_request(guard), &attempts)
5533 .map_err(LifecycleError::Failed)?;
5534 #[cfg(target_os = "macos")]
5535 self.prune_abandoned_seeds(&attempts);
5536 Ok(())
5537 }
5538
5539 #[cfg(target_os = "macos")]
5550 fn prune_abandoned_seeds(&self, attempts: &[RunnerAttempt]) {
5551 let Ok(policies) = self.ports.store.policies() else {
5552 return;
5553 };
5554 let Ok(Some(host_root)) = self.resolvable_host_root() else {
5555 return;
5556 };
5557 let in_use: BTreeSet<PathBuf> = policies
5558 .iter()
5559 .filter_map(|policy| match policy.workspace_policy() {
5560 WorkspacePolicy::Persistent { root } => Some(canonical_or_raw(root.as_path())),
5561 WorkspacePolicy::Ephemeral => None,
5562 })
5563 .chain(std::iter::once(canonical_or_raw(host_root.as_path())))
5564 .collect();
5565 let used: BTreeSet<&Path> = attempts
5567 .iter()
5568 .filter_map(|attempt| attempt.runtime_path().parent())
5569 .collect();
5570 for root in used.into_iter().map(canonical_or_raw) {
5571 if !in_use.contains(&root) {
5572 let _ = remove_runtime_tree(&root.join(PACKAGE_SEED_DIR));
5573 }
5574 }
5575 }
5576}
5577
5578#[cfg(target_os = "macos")]
5579fn canonical_or_raw(path: &Path) -> PathBuf {
5580 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
5581}
5582
5583#[async_trait]
5584impl RunnerLauncher for LifecycleLauncher {
5585 async fn host_capacity(
5586 &self,
5587 host: &runner_manager_domain::model::Host,
5588 ) -> Result<std::num::NonZeroU16, crate::reconcile::HostCapacityError> {
5589 self.ports
5590 .store
5591 .host(host.id)
5592 .map_err(|_| crate::reconcile::HostCapacityError::JournalUnreadable)?
5593 .map(|current| current.host_capacity)
5594 .ok_or(crate::reconcile::HostCapacityError::MissingHost)
5595 }
5596
5597 async fn supervise(
5598 &self,
5599 policy: &ScalePolicy,
5600 ) -> Result<Vec<ReplacementIntent>, LaunchFailure> {
5601 LifecycleLauncher::supervise(self, policy)
5602 .await
5603 .map_err(|error| LaunchFailure::new(error.reason()))
5604 }
5605
5606 async fn attempts(&self) -> Result<Vec<RunnerAttempt>, LaunchFailure> {
5607 self.ports.store.attempts().map_err(|_| {
5608 LaunchFailure::new(FailureReason::Other(
5609 "attempt journal could not be read".into(),
5610 ))
5611 })
5612 }
5613
5614 async fn launch(&self, request: LaunchRequest<'_>) -> Result<RunnerAttempt, LaunchFailure> {
5615 self.reserve_attempt(request.policy, request.allocation_guard)
5616 .map_err(|error| LaunchFailure::new(error.reason()))
5617 }
5618
5619 async fn finish_launch(
5620 &self,
5621 policy: &ScalePolicy,
5622 attempt: RunnerAttempt,
5623 waits: ReservationWaits,
5624 ) -> Result<RunnerAttempt, LaunchFailure> {
5625 self.finish_attempt(policy, attempt, waits)
5626 .await
5627 .map_err(|error| LaunchFailure::new(error.reason()))
5628 }
5629
5630 async fn clean(&self, id: AttemptId) -> Result<(), LaunchFailure> {
5631 let mut attempt = self
5632 .ports
5633 .store
5634 .attempt(id)
5635 .map_err(|_| {
5636 LaunchFailure::new(FailureReason::Other(
5637 "attempt journal could not be read".into(),
5638 ))
5639 })?
5640 .ok_or_else(|| {
5641 LaunchFailure::new(FailureReason::Other(
5642 "attempt disappeared from the journal".into(),
5643 ))
5644 })?;
5645 self.clean_attempt(&mut attempt)
5646 .map_err(|error| LaunchFailure::new(error.reason()))?;
5647 self.cleanup_retries().succeeded(id);
5648 Ok(())
5649 }
5650}
5651
5652#[must_use]
5655pub fn runner_name(attempt: AttemptId) -> String {
5656 format!("runner-manager-{attempt}")
5657}
5658
5659const EVIDENCE_DEADLINE: Duration = Duration::from_secs(2);
5661
5662#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5672enum JobEvidence {
5673 RanAJob,
5674 NoJob,
5675 Unknown,
5676}
5677
5678impl JobEvidence {
5679 fn of(runtime: &Path) -> Self {
5680 let diag = runtime.join("_diag");
5681 if runner_manager_platform::host_fitness::directory_responds(&diag, EVIDENCE_DEADLINE)
5685 != runner_manager_platform::host_fitness::Responsiveness::Responds
5686 {
5687 return Self::Unknown;
5688 }
5689 let Ok(entries) = fs::read_dir(diag) else {
5690 return Self::Unknown;
5691 };
5692 let mut listener = false;
5693 for entry in entries.flatten() {
5694 let name = entry.file_name();
5695 let name = name.to_string_lossy();
5696 if !name.ends_with(".log") {
5697 continue;
5698 }
5699 if name.starts_with("Worker_") {
5700 return Self::RanAJob;
5701 }
5702 listener |= name.starts_with("Runner_");
5703 }
5704 if listener { Self::NoJob } else { Self::Unknown }
5705 }
5706}
5707
5708fn known_runner_id(attempt: &RunnerAttempt, observed: Option<u64>) -> Option<u64> {
5711 attempt
5712 .github_runner_id()
5713 .or(observed)
5714 .or_else(|| read_runner_id(attempt.runtime_path()))
5715}
5716
5717fn read_runner_id(runtime: &Path) -> Option<u64> {
5718 fs::read_to_string(runtime.join(RUNNER_ID_FILE))
5719 .ok()?
5720 .trim()
5721 .parse()
5722 .ok()
5723}
5724
5725fn write_runner_id(runtime: &Path, runner_id: u64) -> Result<(), LifecycleError> {
5726 let target = runtime.join(RUNNER_ID_FILE);
5727 if let Some(existing) = read_runner_id(runtime) {
5728 return (existing == runner_id)
5729 .then_some(())
5730 .ok_or(LifecycleError::Journal);
5731 }
5732 let temporary = runtime.join(format!("{RUNNER_ID_FILE}.{}.tmp", uuid::Uuid::new_v4()));
5733 write_durable_file(&temporary, runner_id.to_string().as_bytes())
5734 .map_err(|_| LifecycleError::Journal)?;
5735 match fs::rename(&temporary, &target) {
5736 Ok(()) => sync_directory(runtime).map_err(|_| LifecycleError::Journal),
5737 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
5738 let _ = fs::remove_file(&temporary);
5739 (read_runner_id(runtime) == Some(runner_id))
5740 .then_some(())
5741 .ok_or(LifecycleError::Journal)
5742 }
5743 Err(_) => {
5744 let _ = fs::remove_file(&temporary);
5745 Err(LifecycleError::Journal)
5746 }
5747 }
5748}
5749
5750fn write_durable_file(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
5751 let mut options = fs::OpenOptions::new();
5752 options.create(true).truncate(true).write(true);
5753 #[cfg(unix)]
5757 std::os::unix::fs::OpenOptionsExt::custom_flags(&mut options, libc::O_NOFOLLOW);
5758 let mut file = options.open(path)?;
5759 file.write_all(bytes)?;
5760 file.sync_all()?;
5761 let parent = path.parent().ok_or_else(|| {
5762 std::io::Error::new(
5763 std::io::ErrorKind::InvalidInput,
5764 "file has no parent directory",
5765 )
5766 })?;
5767 sync_directory(parent)
5768}
5769
5770#[cfg(unix)]
5771fn sync_directory(path: &Path) -> std::io::Result<()> {
5772 fs::File::open(path)?.sync_all()
5773}
5774
5775#[cfg(windows)]
5776fn sync_directory(path: &Path) -> std::io::Result<()> {
5777 use std::os::windows::fs::OpenOptionsExt;
5778
5779 const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
5780 const FILE_SHARE_ALL: u32 = 0x0000_0007;
5781 const GENERIC_WRITE: u32 = 0x4000_0000;
5782 fs::OpenOptions::new()
5783 .access_mode(GENERIC_WRITE)
5784 .share_mode(FILE_SHARE_ALL)
5785 .custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
5786 .open(path)?
5787 .sync_all()
5788}
5789
5790#[cfg(test)]
5791mod tests {
5792 use super::*;
5793
5794 #[cfg(unix)]
5796 fn attempt_owned_in(root: &Path, name: &str) -> RunnerAttempt {
5797 let runtime = root.join(name);
5798 fs::create_dir_all(&runtime).unwrap();
5799 RunnerAttempt::allocate(
5800 AttemptId::new_random(),
5801 PolicyId::from_u128(0xfeed),
5802 &runtime,
5803 FakeClock::default().now(),
5804 )
5805 }
5806
5807 #[cfg(unix)]
5808 #[test]
5809 fn no_two_live_attempts_share_a_runner_account() {
5810 use std::os::unix::fs::MetadataExt as _;
5811
5812 let root = tempfile::tempdir().unwrap();
5815 let me = fs::metadata(root.path()).unwrap();
5816 let mine = RunnerAccount {
5817 uid: me.uid(),
5818 gid: me.gid(),
5819 };
5820 let other = RunnerAccount {
5821 uid: me.uid().wrapping_add(1),
5822 gid: me.gid(),
5823 };
5824 let processes = NativeProcesses {
5825 runner: RunnerIdentity::Pool(RunnerPool {
5826 gid: me.gid(),
5827 accounts: vec![other, mine],
5828 legacy: None,
5829 }),
5830 ..NativeProcesses::default()
5831 };
5832
5833 let first = attempt_owned_in(root.path(), "a");
5834 assert_eq!(
5835 processes.account_for(&first).unwrap(),
5836 Some(mine),
5837 "an attempt an account already owns keeps it"
5838 );
5839 assert_eq!(processes.account_for(&first).unwrap(), Some(mine));
5840 let second = attempt_owned_in(root.path(), "b");
5841 assert_eq!(
5842 processes.account_for(&second).unwrap(),
5843 Some(other),
5844 "the account a live attempt holds is not given to another"
5845 );
5846 let third = attempt_owned_in(root.path(), "c");
5847 assert!(
5848 processes.account_for(&third).is_err(),
5849 "with every account busy no runner starts on a shared one"
5850 );
5851
5852 processes.destroy(&first).unwrap();
5853 fs::remove_dir(first.runtime_path()).unwrap();
5854 assert_eq!(processes.account_for(&third).unwrap(), Some(mine));
5855 }
5856
5857 #[cfg(unix)]
5858 #[test]
5859 fn a_pool_with_only_the_earlier_single_account_shares_it_as_before() {
5860 use std::os::unix::fs::MetadataExt as _;
5861
5862 let root = tempfile::tempdir().unwrap();
5863 let me = fs::metadata(root.path()).unwrap();
5864 let legacy = RunnerAccount {
5865 uid: me.uid(),
5866 gid: me.gid(),
5867 };
5868 let processes = NativeProcesses {
5869 runner: RunnerIdentity::Pool(RunnerPool {
5870 gid: me.gid(),
5871 accounts: Vec::new(),
5872 legacy: Some(legacy),
5873 }),
5874 ..NativeProcesses::default()
5875 };
5876 for name in ["a", "b"] {
5877 assert_eq!(
5878 processes
5879 .account_for(&attempt_owned_in(root.path(), name))
5880 .unwrap(),
5881 Some(legacy)
5882 );
5883 }
5884 }
5885
5886 #[test]
5887 fn a_confined_root_without_the_pool_starts_no_runner() {
5888 let processes = NativeProcesses {
5889 runner: RunnerIdentity::Missing("no pool".into()),
5890 ..NativeProcesses::default()
5891 };
5892 let attempt = RunnerAttempt::allocate(
5893 AttemptId::new_random(),
5894 PolicyId::from_u128(0xfeed),
5895 Path::new("/nonexistent/attempt"),
5896 FakeClock::default().now(),
5897 );
5898 assert_eq!(
5899 processes.account_for(&attempt),
5900 Err(FailureReason::Other("no pool".into()))
5901 );
5902 }
5903 use std::collections::BTreeSet;
5904 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
5905
5906 use crate::reconcile::{AllocationLock, InProcessAllocationLock};
5907 use runner_manager_domain::model::{Elapsed, TargetScope};
5908 use runner_manager_domain::store::SqliteStore;
5909 use runner_manager_github::jit::JitRunner;
5910 use runner_manager_testkit::clock::FakeClock;
5911 use runner_manager_testkit::fixtures;
5912
5913 #[test]
5914 fn production_provider_set_routes_every_implemented_isolated_backend() {
5915 let providers = PlatformExecutionProvider::new(HostId::from_u128(1));
5916 assert!(providers.isolated_provider(Backend::Auto).is_some());
5917 assert!(providers.isolated_provider(Backend::Oci).is_some());
5918 assert!(
5919 providers
5920 .isolated_provider(Backend::WindowsHyperVContainer)
5921 .is_some()
5922 );
5923 assert!(
5924 providers
5925 .isolated_provider(Backend::VirtualMachine)
5926 .is_none()
5927 );
5928 }
5929
5930 type CapturedFields = Vec<(String, String)>;
5932
5933 #[derive(Clone, Default)]
5941 struct CapturedEvents(std::sync::Arc<std::sync::Mutex<Vec<CapturedFields>>>);
5942
5943 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for CapturedEvents {
5944 fn on_event(
5945 &self,
5946 event: &tracing::Event<'_>,
5947 _context: tracing_subscriber::layer::Context<'_, S>,
5948 ) {
5949 struct Collect(Vec<(String, String)>);
5950 impl tracing::field::Visit for Collect {
5951 fn record_debug(
5952 &mut self,
5953 field: &tracing::field::Field,
5954 value: &dyn std::fmt::Debug,
5955 ) {
5956 self.0.push((
5959 field.name().to_owned(),
5960 format!("{value:?}").trim_matches('"').to_owned(),
5961 ));
5962 }
5963 }
5964 let mut collected = Collect(Vec::new());
5965 event.record(&mut collected);
5966 self.0
5967 .lock()
5968 .expect("the capture mutex is not poisoned")
5969 .push(collected.0);
5970 }
5971 }
5972
5973 #[test]
5991 fn a_launch_the_runner_root_refused_names_the_cause_in_the_log_that_ships() {
5992 use runner_manager_platform::logging;
5993 use tracing_subscriber::layer::SubscriberExt as _;
5994
5995 let captured = CapturedEvents::default();
5996 let error = RunnerRootError::DeniedByPrivacyPolicy {
5997 requested: PathBuf::from("/Volumes/NVME/runners"),
5998 refused: PathBuf::from("/Volumes/NVME"),
5999 remediation: RootOwner::Host.remediation(),
6000 };
6001 let kind = error.kind();
6002
6003 let failure = tracing::subscriber::with_default(
6004 tracing_subscriber::registry().with(captured.clone()),
6005 || root_failure(error),
6006 );
6007
6008 assert!(
6013 matches!(
6014 &failure,
6015 LifecycleError::Failed(FailureReason::Other(detail))
6016 if detail.contains("/Volumes/NVME/runners")
6017 && detail.contains("Full Disk Access")
6018 ),
6019 "the reason must still carry the detail: {failure:?}"
6020 );
6021
6022 let events = captured
6023 .0
6024 .lock()
6025 .expect("the capture mutex is not poisoned")
6026 .clone();
6027 let event = events
6028 .iter()
6029 .find(|fields| fields.iter().any(|(_, value)| value.contains(kind)))
6030 .unwrap_or_else(|| panic!("the refusal did not name its cause: {events:?}"));
6031
6032 for (name, value) in event {
6037 assert!(
6038 logging::is_field_allowed(name),
6039 "`{name}` is not allow-listed, so it ships as `{}`: {event:?}",
6040 logging::REDACTION
6041 );
6042 assert_eq!(
6043 &logging::redact(value),
6044 value,
6045 "`{name}` does not survive value-shape scrubbing: {event:?}"
6046 );
6047 }
6048 }
6049
6050 fn nz(slot: u16) -> NonZeroU16 {
6052 NonZeroU16::new(slot).expect("a positive slot")
6053 }
6054
6055 const JIT: &str = "eyJzZWNyZXQiOiJnaHBfRE9fTk9UX0xFQUsifQ==";
6056
6057 #[derive(Debug, Default)]
6058 struct FakeGithubLifecycle {
6059 registration_failures: Mutex<VecDeque<bool>>,
6060 observations: Mutex<VecDeque<LifecycleGithubObservation>>,
6061 registrations: AtomicUsize,
6062 remaining_runners: AtomicUsize,
6063 deregistrations: Mutex<Vec<u64>>,
6067 deregistration_fails: AtomicBool,
6070 journal: Mutex<Option<Arc<SqliteStore>>>,
6075 registration_facts: Mutex<Vec<RegistrationFact>>,
6077 }
6078
6079 #[derive(Debug, Clone)]
6086 struct RegistrationFact {
6087 leased_slots: Vec<u16>,
6089 work_folder: String,
6091 runner_name: String,
6094 journal_states: Vec<AttemptState>,
6095 }
6096
6097 impl FakeGithubLifecycle {
6098 fn fail(mut self, terminal: bool) -> Self {
6099 self.registration_failures
6100 .get_mut()
6101 .expect("unpoisoned")
6102 .push_back(terminal);
6103 self
6104 }
6105
6106 fn watch_journal(&self, store: Arc<SqliteStore>) {
6107 *self.journal.lock().unwrap() = Some(store);
6108 }
6109
6110 fn registration_facts(&self) -> Vec<RegistrationFact> {
6111 self.registration_facts.lock().unwrap().clone()
6112 }
6113
6114 fn observe(&self, observation: GithubRunnerObservation) {
6115 let observation = match observation {
6116 GithubRunnerObservation::Unreachable => LifecycleGithubObservation::unreachable(),
6117 GithubRunnerObservation::NotRegistered => {
6118 LifecycleGithubObservation::not_registered()
6119 }
6120 GithubRunnerObservation::Registered { busy } => {
6121 LifecycleGithubObservation::registered(73, busy)
6122 }
6123 };
6124 self.observations.lock().unwrap().push_back(observation);
6125 }
6126 }
6127
6128 #[async_trait]
6129 impl LifecycleGithub for FakeGithubLifecycle {
6130 async fn register(
6131 &self,
6132 _target: &ScaleTarget,
6133 request: &JitRunnerRequest,
6134 _cancel: &CancelToken,
6135 ) -> Result<JitRegistration, JitRequestFailure> {
6136 self.registrations.fetch_add(1, Ordering::SeqCst);
6137 if let Some(store) = self.journal.lock().unwrap().as_ref() {
6138 let attempts = store.attempts().expect("the journal is readable");
6139 let slots = attempts
6140 .iter()
6141 .filter_map(|attempt| attempt.workspace().slot_number())
6142 .collect();
6143 self.registration_facts
6144 .lock()
6145 .unwrap()
6146 .push(RegistrationFact {
6147 leased_slots: slots,
6148 work_folder: request.work_folder().to_string(),
6149 runner_name: request.name().to_string(),
6150 journal_states: attempts.iter().map(RunnerAttempt::state).collect(),
6151 });
6152 }
6153 if let Some(terminal) = self.registration_failures.lock().unwrap().pop_front() {
6154 return Err(JitRequestFailure {
6155 terminal,
6156 reason: if terminal {
6157 FailureReason::Other("GitHub refused JIT registration with 403".into())
6158 } else {
6159 FailureReason::JitRequestFailed
6160 },
6161 retry_after: None,
6162 });
6163 }
6164 self.remaining_runners.store(1, Ordering::SeqCst);
6165 Ok(JitRegistration::new(
6166 EncodedJitConfig::new(JIT),
6167 JitRunner {
6168 id: 73,
6169 name: request.name().to_string(),
6170 os: "windows".into(),
6171 status: "offline".into(),
6172 busy: false,
6173 runner_group_id: Some(1),
6174 labels: request.labels().to_vec(),
6175 },
6176 ))
6177 }
6178
6179 async fn observe(
6180 &self,
6181 _target: &ScaleTarget,
6182 _attempt: AttemptId,
6183 _cancel: &CancelToken,
6184 ) -> LifecycleGithubObservation {
6185 let observation = self
6186 .observations
6187 .lock()
6188 .unwrap()
6189 .pop_front()
6190 .unwrap_or(LifecycleGithubObservation::not_registered());
6191 if observation.status == GithubRunnerObservation::NotRegistered {
6192 self.remaining_runners.store(0, Ordering::SeqCst);
6193 }
6194 observation
6195 }
6196
6197 async fn deregister(
6198 &self,
6199 _target: &ScaleTarget,
6200 runner_id: u64,
6201 _cancel: &CancelToken,
6202 ) -> bool {
6203 self.deregistrations.lock().unwrap().push(runner_id);
6204 if self.deregistration_fails.load(Ordering::SeqCst) {
6205 return false;
6206 }
6207 self.remaining_runners.store(0, Ordering::SeqCst);
6208 true
6209 }
6210 }
6211
6212 #[derive(Debug)]
6213 struct FakePackages {
6214 version: RunnerVersion,
6215 leases: Mutex<BTreeSet<AttemptId>>,
6216 materializations: AtomicUsize,
6217 materializing: AtomicUsize,
6218 peak_materializing: AtomicUsize,
6219 materialization_failures: AtomicUsize,
6220 release_failures: AtomicUsize,
6221 releases: AtomicUsize,
6222 prunes: AtomicUsize,
6223 }
6224
6225 impl Default for FakePackages {
6226 fn default() -> Self {
6227 Self {
6228 version: RunnerVersion::parse("2.330.0").unwrap(),
6229 leases: Mutex::new(BTreeSet::new()),
6230 materializations: AtomicUsize::new(0),
6231 materializing: AtomicUsize::new(0),
6232 peak_materializing: AtomicUsize::new(0),
6233 materialization_failures: AtomicUsize::new(0),
6234 release_failures: AtomicUsize::new(0),
6235 releases: AtomicUsize::new(0),
6236 prunes: AtomicUsize::new(0),
6237 }
6238 }
6239 }
6240
6241 impl FakePackages {
6242 fn fail_materializations(&self, count: usize) {
6243 self.materialization_failures.store(count, Ordering::SeqCst);
6244 }
6245
6246 fn fail_releases(&self, count: usize) {
6247 self.release_failures.store(count, Ordering::SeqCst);
6248 }
6249 }
6250
6251 #[async_trait]
6252 impl RuntimePackages for FakePackages {
6253 async fn materialize(
6254 &self,
6255 attempt: &RunnerAttempt,
6256 _timings: &mut PackageTimings,
6257 ) -> Result<RunnerVersion, FailureReason> {
6258 self.materializations.fetch_add(1, Ordering::SeqCst);
6259 let now = self.materializing.fetch_add(1, Ordering::SeqCst) + 1;
6260 self.peak_materializing.fetch_max(now, Ordering::SeqCst);
6261 for _ in 0..5 {
6262 tokio::task::yield_now().await;
6263 }
6264 self.materializing.fetch_sub(1, Ordering::SeqCst);
6265 if self
6266 .materialization_failures
6267 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
6268 if left > 0 { Some(left - 1) } else { None }
6269 })
6270 .is_ok()
6271 {
6272 return Err(FailureReason::Other(
6273 "runner package materialization failed transiently".into(),
6274 ));
6275 }
6276 fs::create_dir_all(attempt.runtime_path()).unwrap();
6277 fs::write(attempt.runtime_path().join("runner-package"), b"verified").unwrap();
6278 self.leases.lock().unwrap().insert(attempt.id);
6279 Ok(self.version.clone())
6280 }
6281
6282 fn release(&self, attempt: AttemptId) -> Result<(), FailureReason> {
6283 if self
6284 .release_failures
6285 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
6286 if left > 0 { Some(left - 1) } else { None }
6287 })
6288 .is_ok()
6289 {
6290 return Err(FailureReason::Other(
6291 "runner package lease could not be released".into(),
6292 ));
6293 }
6294 self.leases.lock().unwrap().remove(&attempt);
6295 self.releases.fetch_add(1, Ordering::SeqCst);
6296 Ok(())
6297 }
6298
6299 fn prune_obsolete_guarded(
6300 &self,
6301 _authority: PruneAuthority<'_>,
6302 _attempts: &[RunnerAttempt],
6303 ) -> Result<(), FailureReason> {
6304 self.prunes.fetch_add(1, Ordering::SeqCst);
6305 Ok(())
6306 }
6307 }
6308
6309 #[derive(Debug, Default)]
6310 struct FakeProcesses {
6311 alive: AtomicBool,
6312 completed_successfully: AtomicBool,
6313 spawns: AtomicUsize,
6314 spawn_failures: AtomicUsize,
6315 live_spawn_failure: AtomicBool,
6316 terminations: AtomicUsize,
6317 intent: AtomicBool,
6318 intent_failure: AtomicBool,
6319 actions: Mutex<Vec<&'static str>>,
6320 saw_secret: AtomicBool,
6321 }
6322
6323 impl FakeProcesses {
6324 fn fail_spawns(&self, count: usize) {
6325 self.spawn_failures.store(count, Ordering::SeqCst);
6326 }
6327
6328 fn fail_spawn_with_live_child(&self) {
6329 self.live_spawn_failure.store(true, Ordering::SeqCst);
6330 }
6331
6332 fn set_alive(&self, alive: bool) {
6333 self.alive.store(alive, Ordering::SeqCst);
6334 }
6335
6336 fn finish_successfully(&self) {
6337 self.completed_successfully.store(true, Ordering::SeqCst);
6338 self.alive.store(false, Ordering::SeqCst);
6339 }
6340
6341 fn fail_intent(&self) {
6342 self.intent_failure.store(true, Ordering::SeqCst);
6343 }
6344 }
6345
6346 impl ProcessSupervisor for FakeProcesses {
6347 fn spawn(
6348 &self,
6349 attempt: &RunnerAttempt,
6350 config: &EncodedJitConfig,
6351 ) -> Result<u32, ProcessStartFailure> {
6352 self.spawns.fetch_add(1, Ordering::SeqCst);
6353 let handoff = RestrictiveHandoff::create(
6356 attempt.runtime_path(),
6357 SecretString::from(config.expose().to_owned()),
6358 )
6359 .unwrap();
6360 self.saw_secret
6361 .store(config.expose() == JIT, Ordering::SeqCst);
6362 let handoff_path = handoff.path().to_path_buf();
6363 let failing = self
6364 .spawn_failures
6365 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
6366 if left > 0 { Some(left - 1) } else { None }
6367 })
6368 .is_ok();
6369 drop(handoff);
6370 assert!(!handoff_path.exists(), "handoff must be absent on return");
6371 if self.live_spawn_failure.swap(false, Ordering::SeqCst) {
6372 self.alive.store(true, Ordering::SeqCst);
6373 return Err(ProcessStartFailure::after_spawn_live(4242));
6374 }
6375 if failing {
6376 return Err(ProcessStartFailure::before_spawn(
6377 FailureReason::ProcessStartFailed,
6378 ));
6379 }
6380 self.alive.store(true, Ordering::SeqCst);
6381 Ok(4242)
6382 }
6383
6384 fn is_alive(&self, _attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
6385 self.actions.lock().unwrap().push("observe_process");
6386 Ok(self.alive.load(Ordering::SeqCst))
6387 }
6388
6389 fn recovered_pid(&self, _attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
6390 Ok(self.alive.load(Ordering::SeqCst).then_some(4242))
6391 }
6392
6393 fn completed_successfully(&self, _attempt: &RunnerAttempt) -> bool {
6394 self.completed_successfully.load(Ordering::SeqCst)
6395 }
6396
6397 fn record_terminate_intent(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6398 self.actions.lock().unwrap().push("terminate_intent");
6399 if self.intent_failure.load(Ordering::SeqCst) {
6400 return Err(FailureReason::Other(
6401 "terminate intent directory sync failed".into(),
6402 ));
6403 }
6404 self.intent.store(true, Ordering::SeqCst);
6405 Ok(())
6406 }
6407
6408 fn has_terminate_intent(&self, _attempt: &RunnerAttempt) -> bool {
6409 self.intent.load(Ordering::SeqCst)
6410 }
6411
6412 fn terminate(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6413 assert!(
6414 self.intent.load(Ordering::SeqCst),
6415 "the durable intent must exist before signalling"
6416 );
6417 self.actions.lock().unwrap().push("terminate");
6418 self.terminations.fetch_add(1, Ordering::SeqCst);
6419 self.alive.store(false, Ordering::SeqCst);
6420 Ok(())
6421 }
6422 }
6423
6424 #[derive(Debug, Default)]
6425 struct FakeIsolatedProvider {
6426 resources: Mutex<BTreeMap<AttemptId, (EnvironmentIdentity, EnvironmentState)>>,
6427 extra_resources: Mutex<Vec<EnvironmentIdentity>>,
6428 unsupported_host: AtomicBool,
6429 permission_denied: AtomicBool,
6430 incompatible_image: AtomicBool,
6431 prepare_failure: AtomicBool,
6432 recovery_unavailable: AtomicBool,
6433 destroy_deferred: AtomicBool,
6434 handoffs: AtomicUsize,
6435 completed_successfully: AtomicBool,
6436 native_calls: AtomicUsize,
6437 actions: Mutex<Vec<&'static str>>,
6438 }
6439
6440 impl FakeIsolatedProvider {
6441 fn identity(attempt: &RunnerAttempt, host: HostId) -> EnvironmentIdentity {
6442 let AttemptExecution::Isolated {
6443 provider_kind,
6444 resolved_image,
6445 generation,
6446 ..
6447 } = attempt.execution()
6448 else {
6449 panic!("isolated intent required")
6450 };
6451 EnvironmentIdentity::Isolated {
6452 host,
6453 attempt: attempt.id,
6454 provider_kind: *provider_kind,
6455 environment_id: format!("environment-{}", attempt.id),
6456 resolved_image: resolved_image.clone(),
6457 generation: generation.clone(),
6458 }
6459 }
6460
6461 fn resource_count(&self) -> usize {
6462 self.resources.lock().unwrap().len()
6463 }
6464 }
6465
6466 impl ExecutionProvider for FakeIsolatedProvider {
6467 fn probe(&self, policy: &ScalePolicy) -> ProviderCapability {
6468 if self.unsupported_host.load(Ordering::SeqCst) {
6469 ProviderCapability::Unsupported
6470 } else if self.permission_denied.load(Ordering::SeqCst) {
6471 ProviderCapability::PermissionDenied
6472 } else if policy.execution_policy().is_native() {
6473 ProviderCapability::Unsupported
6474 } else {
6475 ProviderCapability::Ready
6476 }
6477 }
6478
6479 fn resolve(
6480 &self,
6481 policy: &ScalePolicy,
6482 ) -> Result<Option<ResolvedEnvironment>, FailureReason> {
6483 let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
6484 policy.execution_policy()
6485 else {
6486 return Err(FailureReason::Other("isolated policy required".into()));
6487 };
6488 Ok(Some(ResolvedEnvironment {
6489 provider_kind: Backend::Oci,
6490 image: if self.incompatible_image.load(Ordering::SeqCst) {
6491 ImageReference::new("vm-version:wrong-provider")
6492 .expect("valid but incompatible")
6493 } else {
6494 image.clone()
6495 },
6496 }))
6497 }
6498
6499 fn prepare(
6500 &self,
6501 attempt: &RunnerAttempt,
6502 policy: &ScalePolicy,
6503 ) -> Result<PreparedEnvironment, FailureReason> {
6504 assert_eq!(
6505 attempt.state(),
6506 AttemptState::Preparing,
6507 "intent must be durable before provider effect"
6508 );
6509 self.actions.lock().unwrap().push("prepare");
6510 let identity = Self::identity(attempt, policy.host_id);
6511 self.resources
6512 .lock()
6513 .unwrap()
6514 .insert(attempt.id, (identity.clone(), EnvironmentState::Starting));
6515 if self.prepare_failure.swap(false, Ordering::SeqCst) {
6516 return Err(FailureReason::Other("provider-secret-needle".into()));
6517 }
6518 Ok(PreparedEnvironment::isolated(attempt.id, identity))
6519 }
6520
6521 fn start(
6522 &self,
6523 prepared: PreparedEnvironment,
6524 attempt: &RunnerAttempt,
6525 handoff: OneTimeJitHandoff<'_>,
6526 ) -> Result<EnvironmentIdentity, ProcessStartFailure> {
6527 assert_eq!(prepared.attempt, attempt.id);
6528 assert_eq!(attempt.state(), AttemptState::JitReceived);
6529 assert_eq!(handoff.consume().expose(), JIT);
6530 self.handoffs.fetch_add(1, Ordering::SeqCst);
6531 self.actions.lock().unwrap().push("start");
6532 let mut resources = self.resources.lock().unwrap();
6533 let (identity, state) = resources.get_mut(&attempt.id).expect("prepared resource");
6534 *state = EnvironmentState::Running;
6535 Ok(identity.clone())
6536 }
6537
6538 fn inspect(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
6539 self.actions.lock().unwrap().push("inspect");
6540 Ok(self
6541 .resources
6542 .lock()
6543 .unwrap()
6544 .get(&attempt.id)
6545 .map_or(EnvironmentState::Missing, |(_, state)| *state))
6546 }
6547
6548 fn recover(&self, attempt: &RunnerAttempt) -> Result<EnvironmentState, FailureReason> {
6549 if self.recovery_unavailable.load(Ordering::SeqCst) {
6550 return Err(FailureReason::Other(
6551 "provider discovery unavailable".into(),
6552 ));
6553 }
6554 self.inspect(attempt)
6555 }
6556
6557 fn owns(&self, attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
6558 Ok(self
6559 .resources
6560 .lock()
6561 .unwrap()
6562 .get(&attempt.id)
6563 .is_some_and(|(identity, _)| {
6564 identity_matches_intent(attempt_host(identity), attempt, identity)
6565 }))
6566 }
6567
6568 fn stop(&self, attempt: &RunnerAttempt) -> Result<ProviderStop, FailureReason> {
6569 assert!(self.owns(attempt)?);
6570 self.actions.lock().unwrap().push("stop");
6571 if let Some((_, state)) = self.resources.lock().unwrap().get_mut(&attempt.id) {
6572 *state = EnvironmentState::Exited;
6573 }
6574 Ok(ProviderStop::Stopped)
6575 }
6576
6577 fn destroy(&self, attempt: &RunnerAttempt) -> Result<ProviderDestroy, FailureReason> {
6578 assert!(self.owns(attempt)?);
6579 self.actions.lock().unwrap().push("destroy");
6580 if self.destroy_deferred.load(Ordering::SeqCst) {
6581 return Ok(ProviderDestroy::Deferred("provider busy"));
6582 }
6583 self.resources.lock().unwrap().remove(&attempt.id);
6584 Ok(ProviderDestroy::Destroyed)
6585 }
6586
6587 fn enumerate_owned(&self, host: HostId) -> Vec<EnvironmentIdentity> {
6588 let mut owned = self
6589 .resources
6590 .lock()
6591 .unwrap()
6592 .values()
6593 .filter(|(identity, _)| attempt_host(identity) == host)
6594 .map(|(identity, _)| identity.clone())
6595 .collect::<Vec<_>>();
6596 owned.extend(
6597 self.extra_resources
6598 .lock()
6599 .unwrap()
6600 .iter()
6601 .filter(|identity| attempt_host(identity) == host)
6602 .cloned(),
6603 );
6604 owned
6605 }
6606
6607 fn diagnostics(&self, _attempt: &RunnerAttempt) -> Vec<ProviderDiagnostic> {
6608 vec![ProviderDiagnostic::CleanupDeferred; 40]
6609 }
6610
6611 fn spawn(
6612 &self,
6613 _attempt: &RunnerAttempt,
6614 _config: &EncodedJitConfig,
6615 ) -> Result<u32, ProcessStartFailure> {
6616 self.native_calls.fetch_add(1, Ordering::SeqCst);
6617 panic!("isolated provider must never spawn natively")
6618 }
6619 fn is_alive(&self, _attempt: &RunnerAttempt) -> Result<bool, FailureReason> {
6620 panic!("native fallback")
6621 }
6622 fn recovered_pid(&self, _attempt: &RunnerAttempt) -> Result<Option<u32>, FailureReason> {
6623 panic!("native fallback")
6624 }
6625 fn completed_successfully(&self, _attempt: &RunnerAttempt) -> bool {
6626 self.completed_successfully.load(Ordering::SeqCst)
6627 }
6628 fn record_terminate_intent(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6629 panic!("native fallback")
6630 }
6631 fn has_terminate_intent(&self, _attempt: &RunnerAttempt) -> bool {
6632 false
6633 }
6634 fn terminate(&self, _attempt: &RunnerAttempt) -> Result<(), FailureReason> {
6635 panic!("native fallback")
6636 }
6637 }
6638
6639 fn attempt_host(identity: &EnvironmentIdentity) -> HostId {
6640 let EnvironmentIdentity::Isolated { host, .. } = identity else {
6641 panic!("isolated identity")
6642 };
6643 *host
6644 }
6645
6646 #[derive(Debug, Default)]
6647 struct FakeDemand {
6648 answers: Mutex<VecDeque<bool>>,
6649 }
6650
6651 impl FakeDemand {
6652 fn answering(answers: impl IntoIterator<Item = bool>) -> Self {
6653 Self {
6654 answers: Mutex::new(answers.into_iter().collect()),
6655 }
6656 }
6657 }
6658
6659 #[async_trait]
6660 impl DemandPersistence for FakeDemand {
6661 async fn persists(&self, _policy: PolicyId) -> bool {
6662 self.answers.lock().unwrap().pop_front().unwrap_or(true)
6663 }
6664 }
6665
6666 #[derive(Debug, Default)]
6667 struct FakeDelay(Mutex<Vec<Duration>>);
6668
6669 #[async_trait]
6670 impl RetryDelay for FakeDelay {
6671 async fn wait(&self, duration: Duration) {
6672 self.0.lock().unwrap().push(duration);
6673 }
6674 }
6675
6676 #[tokio::test]
6677 async fn live_capacity_reads_the_updated_sqlite_host_and_rejects_a_missing_host() {
6678 let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
6679 let path = harness._root.path().join("live-capacity.sqlite3");
6680 harness.store = Arc::new(SqliteStore::open(&path).unwrap());
6681 harness.store.put_host(&harness.host).unwrap();
6682 harness.launcher = harness.restart();
6683 assert_eq!(
6684 harness.launcher.host_capacity(&harness.host).await.unwrap(),
6685 harness.host.host_capacity
6686 );
6687 let writer = SqliteStore::open(&path).unwrap();
6690 let mut updated = harness.host.clone();
6691 updated.host_capacity = std::num::NonZeroU16::new(12).unwrap();
6692 writer.put_host(&updated).unwrap();
6693 assert_eq!(
6694 harness
6695 .launcher
6696 .host_capacity(&harness.host)
6697 .await
6698 .unwrap()
6699 .get(),
6700 12
6701 );
6702
6703 updated.host_capacity = std::num::NonZeroU16::new(2).unwrap();
6704 writer.put_host(&updated).unwrap();
6705 assert_eq!(
6706 harness
6707 .launcher
6708 .host_capacity(&harness.host)
6709 .await
6710 .unwrap()
6711 .get(),
6712 2
6713 );
6714
6715 let mut missing = harness.host.clone();
6716 missing.id = HostId::from_u128(999);
6717 assert_eq!(
6718 harness.launcher.host_capacity(&missing).await,
6719 Err(crate::reconcile::HostCapacityError::MissingHost)
6720 );
6721 }
6722
6723 struct Harness {
6724 _root: tempfile::TempDir,
6725 app_paths: runner_manager_platform::paths::AppPaths,
6726 launcher: LifecycleLauncher,
6727 demand: Arc<dyn DemandPersistence>,
6728 store: Arc<SqliteStore>,
6729 github: Arc<FakeGithubLifecycle>,
6730 packages: Arc<FakePackages>,
6731 processes: Arc<FakeProcesses>,
6732 clock: Arc<FakeClock>,
6733 events: Arc<AttemptEventLog>,
6734 reconcile_events: Arc<crate::reconcile::EventLog>,
6735 delay: Arc<FakeDelay>,
6736 host: runner_manager_domain::model::Host,
6737 policy: ScalePolicy,
6738 allocation_lock: InProcessAllocationLock,
6739 workspace_root: Option<LocalAbsolutePath>,
6741 }
6742
6743 impl Harness {
6744 fn new(github: FakeGithubLifecycle, demand: Arc<dyn DemandPersistence>) -> Self {
6745 let root = tempfile::tempdir().unwrap();
6746 let paths = runner_manager_platform::paths::AppPaths::rooted_at(root.path());
6747 paths.create_all().unwrap();
6748 let policy = fixtures::policy()
6749 .repository("octo/repo")
6750 .autoscale("home", 2)
6751 .active()
6752 .build();
6753 let host_root = root.path().join("host-root");
6759 fs::create_dir_all(&host_root).unwrap();
6760 let mut host = fixtures::host().build();
6761 host.runner_root_override = Some(
6762 LocalAbsolutePath::new(host_root.to_str().expect("a UTF-8 temporary path"))
6763 .expect("a local absolute host root"),
6764 );
6765 let store = Arc::new(SqliteStore::open_in_memory().unwrap());
6766 store.put_host(&host).unwrap();
6767 let github = Arc::new(github);
6768 let packages = Arc::new(FakePackages::default());
6769 let processes = Arc::new(FakeProcesses::default());
6770 let clock = Arc::new(FakeClock::default());
6771 let events = Arc::new(AttemptEventLog::default());
6772 let reconcile_events = Arc::new(crate::reconcile::EventLog::new());
6773 let delay = Arc::new(FakeDelay::default());
6774 let ports = LifecyclePorts {
6775 store: Arc::clone(&store) as Arc<dyn Store>,
6776 github: Arc::clone(&github) as Arc<dyn LifecycleGithub>,
6777 packages: Arc::clone(&packages) as Arc<dyn RuntimePackages>,
6778 processes: Arc::clone(&processes) as Arc<dyn ProcessSupervisor>,
6779 clock: Arc::clone(&clock) as Arc<dyn Clock>,
6780 demand: Arc::clone(&demand),
6781 delay: Arc::clone(&delay) as Arc<dyn RetryDelay>,
6782 events: Arc::clone(&events) as Arc<dyn AttemptEventSink>,
6783 reconcile_events: Arc::clone(&reconcile_events) as Arc<dyn EventSink>,
6784 };
6785 let launcher = Self::launcher_over(policy.host_id, &paths, ports);
6786 Self {
6787 _root: root,
6788 app_paths: paths,
6789 launcher,
6790 demand,
6791 store,
6792 github,
6793 packages,
6794 processes,
6795 clock,
6796 events,
6797 reconcile_events,
6798 delay,
6799 host,
6800 policy,
6801 allocation_lock: InProcessAllocationLock::new(),
6802 workspace_root: None,
6803 }
6804 }
6805
6806 fn with_host_runner_root(mut self) -> Self {
6813 let host_root = self.host_root();
6814 fs::create_dir_all(&host_root).unwrap();
6815 self.host.runner_root_override = Some(
6816 LocalAbsolutePath::new(host_root.to_str().expect("a UTF-8 temporary path"))
6817 .expect("a local absolute host root"),
6818 );
6819 self.store.put_host(&self.host).unwrap();
6820 self
6821 }
6822
6823 fn with_persistent_workspace(mut self, capacity: u16) -> Self {
6825 self = self.with_host_runner_root();
6826 let root = self._root.path().join("persist");
6827 let root = LocalAbsolutePath::new(root.to_str().expect("a UTF-8 temporary path"))
6828 .expect("a local absolute workspace root");
6829 self.policy = fixtures::policy()
6830 .repository("octo/repo")
6831 .autoscale("home", capacity)
6832 .active()
6833 .build();
6834 self.policy
6835 .set_workspace_policy(
6836 WorkspacePolicy::persistent(root.clone(), TargetScope::Repository)
6837 .expect("a repository may be persistent"),
6838 )
6839 .expect("a repository may be persistent");
6840 self.workspace_root = Some(root);
6841 self.store.insert_policy(&self.policy).unwrap();
6844 self
6845 }
6846
6847 fn workspace_root(&self) -> &LocalAbsolutePath {
6848 self.workspace_root
6849 .as_ref()
6850 .expect("this harness configured a persistent workspace")
6851 }
6852
6853 fn slot_path(&self, slot: u16) -> PathBuf {
6854 self.workspace_root().as_path().join(format!("s{slot}"))
6855 }
6856
6857 fn host_root(&self) -> PathBuf {
6858 self._root.path().join("host-root")
6859 }
6860
6861 fn attempt(&self, id: AttemptId) -> RunnerAttempt {
6862 self.store
6863 .attempt(id)
6864 .unwrap()
6865 .expect("the attempt is journalled")
6866 }
6867
6868 fn conclude(&self, id: AttemptId) -> RunnerAttempt {
6870 let mut attempt = self.attempt(id);
6871 attempt
6872 .conclude(
6873 AttemptOutcome::failed(FailureReason::ProcessExitedUnexpectedly),
6874 self.clock.now(),
6875 )
6876 .unwrap();
6877 self.store.record_attempt(&attempt).unwrap();
6878 attempt
6879 }
6880
6881 async fn cleanup_retaining_work(&self, id: AttemptId) {
6882 self.conclude(id);
6883 self.launcher
6884 .clean(id)
6885 .await
6886 .expect("the slot is scrubbed and the lease released");
6887 }
6888
6889 fn launcher_over(
6892 host: HostId,
6893 paths: &runner_manager_platform::paths::AppPaths,
6894 ports: LifecyclePorts,
6895 ) -> LifecycleLauncher {
6896 LifecycleLauncher::new(
6897 host,
6898 paths.clone(),
6899 paths.logs_dir(),
6900 1,
6901 RecoveryTimeouts::new(
6902 Elapsed::seconds(10),
6903 Elapsed::seconds(10),
6904 Elapsed::seconds(10),
6905 ),
6906 RetryPolicy::bounded(3, Duration::from_millis(10), Duration::from_millis(25)),
6907 ports,
6908 )
6909 }
6910
6911 fn restart(&self) -> LifecycleLauncher {
6914 Self::launcher_over(
6915 self.policy.host_id,
6916 &self.app_paths,
6917 LifecyclePorts {
6918 store: Arc::clone(&self.store) as Arc<dyn Store>,
6919 github: Arc::clone(&self.github) as Arc<dyn LifecycleGithub>,
6920 packages: Arc::clone(&self.packages) as Arc<dyn RuntimePackages>,
6921 processes: Arc::clone(&self.processes) as Arc<dyn ProcessSupervisor>,
6922 clock: Arc::clone(&self.clock) as Arc<dyn Clock>,
6923 demand: Arc::clone(&self.demand),
6924 delay: Arc::clone(&self.delay) as Arc<dyn RetryDelay>,
6925 events: Arc::clone(&self.events) as Arc<dyn AttemptEventSink>,
6926 reconcile_events: Arc::clone(&self.reconcile_events) as Arc<dyn EventSink>,
6927 },
6928 )
6929 }
6930
6931 async fn ready(&self) {
6932 self.launcher
6933 .recover_startup(std::slice::from_ref(&self.policy))
6934 .await
6935 .unwrap();
6936 }
6937
6938 async fn launch(&self) -> RunnerAttempt {
6939 self.launch_result().await.unwrap()
6940 }
6941
6942 async fn launch_result(&self) -> Result<RunnerAttempt, LaunchFailure> {
6943 self.launch_after_waiting(Duration::ZERO).await
6944 }
6945
6946 async fn launch_after_waiting(
6947 &self,
6948 lock_wait: Duration,
6949 ) -> Result<RunnerAttempt, LaunchFailure> {
6950 let reserved = self.reserve().await?;
6951 self.launcher
6952 .finish_launch(
6953 &self.policy,
6954 reserved,
6955 ReservationWaits {
6956 lock: lock_wait,
6957 order: Duration::from_secs(2),
6958 },
6959 )
6960 .await
6961 }
6962
6963 async fn launch_through(
6965 &self,
6966 launcher: &LifecycleLauncher,
6967 ) -> Result<RunnerAttempt, LaunchFailure> {
6968 let reserved = self.reserve_through(launcher).await?;
6969 launcher
6970 .finish_launch(&self.policy, reserved, ReservationWaits::default())
6971 .await
6972 }
6973
6974 async fn reserve(&self) -> Result<RunnerAttempt, LaunchFailure> {
6975 self.reserve_through(&self.launcher).await
6976 }
6977
6978 async fn reserve_through(
6979 &self,
6980 launcher: &LifecycleLauncher,
6981 ) -> Result<RunnerAttempt, LaunchFailure> {
6982 let guard = self.allocation_lock.acquire().await.unwrap();
6983 launcher
6984 .launch(LaunchRequest {
6985 host: &self.host,
6986 policy: &self.policy,
6987 allocation_guard: &guard,
6988 })
6989 .await
6990 }
6991
6992 fn only_attempt(&self) -> RunnerAttempt {
6993 self.store.attempts().unwrap().into_iter().next().unwrap()
6994 }
6995 }
6996
6997 fn isolated_harness() -> Harness {
6998 use runner_manager_domain::execution::{ExecutionPolicy, ResourceLimits};
6999 let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
7000 .with_host_runner_root();
7001 let image =
7002 ImageReference::new(format!("registry.example/runner@sha256:{}", "a".repeat(64)))
7003 .expect("pinned image");
7004 harness
7005 .policy
7006 .set_execution_policy(ExecutionPolicy::Isolated {
7007 backend: Backend::Oci,
7008 image,
7009 resources: ResourceLimits {
7010 cpu_millis: 1000,
7011 memory_mib: 1024,
7012 disk_mib: 4096,
7013 },
7014 })
7015 .expect("isolated policy");
7016 harness.github.watch_journal(Arc::clone(&harness.store));
7017 harness
7018 }
7019
7020 fn launcher_with_isolated(
7021 harness: &Harness,
7022 provider: Arc<FakeIsolatedProvider>,
7023 ) -> LifecycleLauncher {
7024 Harness::launcher_over(
7025 harness.policy.host_id,
7026 &harness.app_paths,
7027 LifecyclePorts {
7028 store: Arc::clone(&harness.store) as Arc<dyn Store>,
7029 github: Arc::clone(&harness.github) as Arc<dyn LifecycleGithub>,
7030 packages: Arc::clone(&harness.packages) as Arc<dyn RuntimePackages>,
7031 processes: provider as Arc<dyn ExecutionProvider>,
7032 clock: Arc::clone(&harness.clock) as Arc<dyn Clock>,
7033 demand: Arc::clone(&harness.demand),
7034 delay: Arc::clone(&harness.delay) as Arc<dyn RetryDelay>,
7035 events: Arc::clone(&harness.events) as Arc<dyn AttemptEventSink>,
7036 reconcile_events: Arc::clone(&harness.reconcile_events) as Arc<dyn EventSink>,
7037 },
7038 )
7039 }
7040
7041 #[tokio::test]
7042 async fn isolated_policy_and_journal_never_enter_native_provider_or_jit() {
7043 use runner_manager_domain::execution::{
7044 AttemptExecution, Backend, ExecutionPolicy, ImageReference, ResourceLimits,
7045 };
7046
7047 let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
7048 .with_host_runner_root();
7049 let image =
7050 ImageReference::new(format!("registry.example/runner@sha256:{}", "a".repeat(64)))
7051 .expect("pinned image");
7052 harness
7053 .policy
7054 .set_execution_policy(ExecutionPolicy::Isolated {
7055 backend: Backend::Oci,
7056 image: image.clone(),
7057 resources: ResourceLimits {
7058 cpu_millis: 1000,
7059 memory_mib: 1024,
7060 disk_mib: 4096,
7061 },
7062 })
7063 .expect("valid isolated profile");
7064
7065 harness.ready().await;
7066 assert_eq!(
7067 harness.processes.probe(&harness.policy),
7068 ProviderCapability::Unsupported
7069 );
7070 let failure = harness
7071 .launch_result()
7072 .await
7073 .expect_err("provider is unavailable");
7074 assert_eq!(
7075 failure.reason,
7076 FailureReason::IsolationProvider(IsolationProviderFailure::Unsupported)
7077 );
7078
7079 let runtime = harness.host_root().join("isolated-owned-environment");
7080 fs::create_dir_all(&runtime).expect("provider resource fixture");
7081 fs::write(runtime.join("owned-marker"), "non-secret").expect("fixture marker");
7082 let mut attempt = RunnerAttempt::allocate(
7083 AttemptId::from_u128(0x123),
7084 harness.policy.id,
7085 runtime.clone(),
7086 harness.clock.now(),
7087 );
7088 attempt
7089 .allocate_execution(AttemptExecution::Isolated {
7090 provider_kind: Backend::Oci,
7091 environment_id: None,
7092 resolved_image: image,
7093 generation: "generation-123".into(),
7094 })
7095 .expect("isolated intent");
7096 attempt
7097 .prepared_environment("environment-123".into())
7098 .expect("prepared identity");
7099 harness
7100 .store
7101 .record_attempt(&attempt)
7102 .expect("durable isolated journal");
7103
7104 harness
7105 .launcher
7106 .recover_startup(std::slice::from_ref(&harness.policy))
7107 .await
7108 .expect("unsupported isolated cleanup is profile-local");
7109 let failure = harness
7110 .launch_result()
7111 .await
7112 .expect_err("provider unavailable for this policy");
7113 assert_eq!(
7114 failure.reason,
7115 FailureReason::IsolationProvider(IsolationProviderFailure::Unsupported)
7116 );
7117 harness
7118 .launcher
7119 .supervise(&harness.policy)
7120 .await
7121 .expect("unavailable attempt is deferred");
7122 assert!(harness.launcher.clean(attempt.id).await.is_err());
7123 assert_eq!(
7124 harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7125 AttemptState::CleanupDeferred
7126 );
7127 assert!(matches!(
7128 harness.launcher.recover_startup(&[]).await,
7129 Err(LifecycleError::RecoveryIncomplete)
7130 ));
7131 assert!(
7132 runtime.join("owned-marker").is_file(),
7133 "native cleanup touched the isolated environment"
7134 );
7135 assert!(
7136 harness.processes.actions.lock().unwrap().is_empty(),
7137 "native recovery inspected an isolated identity"
7138 );
7139 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
7140 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7141 assert_eq!(harness.packages.materializations.load(Ordering::SeqCst), 0);
7142 }
7143
7144 #[tokio::test]
7145 async fn isolated_prepare_is_durable_before_jit_and_start_uses_one_handoff() {
7146 let harness = isolated_harness();
7147 let provider = Arc::new(FakeIsolatedProvider::default());
7148 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7149 launcher
7150 .recover_startup(std::slice::from_ref(&harness.policy))
7151 .await
7152 .expect("ready");
7153 let attempt = harness
7154 .launch_through(&launcher)
7155 .await
7156 .expect("isolated launch");
7157 assert_eq!(attempt.state(), AttemptState::Starting);
7158 assert_eq!(provider.resource_count(), 1);
7159 assert_eq!(provider.handoffs.load(Ordering::SeqCst), 1);
7160 assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7161 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 1);
7162 let fact = harness
7163 .github
7164 .registration_facts()
7165 .pop()
7166 .expect("JIT registration");
7167 assert_eq!(
7168 fact.journal_states,
7169 vec![AttemptState::Prepared],
7170 "provider readiness preceded GitHub JIT"
7171 );
7172 assert_eq!(
7173 harness
7174 .events
7175 .events()
7176 .into_iter()
7177 .filter_map(|event| match event {
7178 AttemptEvent::State { state, .. } => Some(state),
7179 _ => None,
7180 })
7181 .collect::<Vec<_>>(),
7182 vec![
7183 AttemptState::Allocated,
7184 AttemptState::Preparing,
7185 AttemptState::Prepared,
7186 AttemptState::JitReceived,
7187 AttemptState::Starting
7188 ]
7189 );
7190 let restarted = launcher_with_isolated(&harness, Arc::clone(&provider));
7191 restarted
7192 .recover_startup(std::slice::from_ref(&harness.policy))
7193 .await
7194 .expect("adopt live sandbox");
7195 assert_eq!(provider.resource_count(), 1);
7196 assert_eq!(
7197 harness.github.registrations.load(Ordering::SeqCst),
7198 1,
7199 "restart cannot issue duplicate JIT"
7200 );
7201 harness
7202 .github
7203 .observe(GithubRunnerObservation::Registered { busy: true });
7204 restarted
7205 .supervise(&harness.policy)
7206 .await
7207 .expect("job observed");
7208 assert_eq!(
7209 harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7210 AttemptState::Busy
7211 );
7212 provider
7213 .resources
7214 .lock()
7215 .unwrap()
7216 .get_mut(&attempt.id)
7217 .unwrap()
7218 .1 = EnvironmentState::Exited;
7219 harness
7220 .github
7221 .observe(GithubRunnerObservation::NotRegistered);
7222 restarted
7223 .supervise(&harness.policy)
7224 .await
7225 .expect("terminal cleanup");
7226 assert_eq!(provider.resource_count(), 0);
7227 assert_eq!(
7228 harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7229 AttemptState::Cleaned
7230 );
7231 }
7232
7233 #[tokio::test]
7234 async fn attached_isolated_job_exit_is_recorded_as_completed() {
7235 let harness = isolated_harness();
7236 let provider = Arc::new(FakeIsolatedProvider::default());
7237 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7238 launcher
7239 .recover_startup(std::slice::from_ref(&harness.policy))
7240 .await
7241 .unwrap();
7242 let attempt = harness.launch_through(&launcher).await.unwrap();
7243 harness
7244 .github
7245 .observe(GithubRunnerObservation::Registered { busy: true });
7246 launcher.supervise(&harness.policy).await.unwrap();
7247 provider
7248 .resources
7249 .lock()
7250 .unwrap()
7251 .get_mut(&attempt.id)
7252 .unwrap()
7253 .1 = EnvironmentState::Exited;
7254 provider
7255 .completed_successfully
7256 .store(true, Ordering::SeqCst);
7257 harness
7258 .github
7259 .observe(GithubRunnerObservation::NotRegistered);
7260 launcher.supervise(&harness.policy).await.unwrap();
7261 let cleaned = harness.store.attempt(attempt.id).unwrap().unwrap();
7262 assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
7263 assert_eq!(cleaned.state(), AttemptState::Cleaned);
7264 }
7265
7266 #[tokio::test]
7267 async fn isolated_prepare_failure_stays_local_and_crash_gap_recovers_owned_resource() {
7268 let harness = isolated_harness();
7269 let provider = Arc::new(FakeIsolatedProvider::default());
7270 provider.prepare_failure.store(true, Ordering::SeqCst);
7271 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7272 launcher
7273 .recover_startup(std::slice::from_ref(&harness.policy))
7274 .await
7275 .expect("ready");
7276 let failure = harness
7277 .launch_through(&launcher)
7278 .await
7279 .expect_err("prepare failed after creating resource");
7280 assert_eq!(
7281 failure.reason,
7282 FailureReason::IsolationProvider(IsolationProviderFailure::RuntimeOperationFailed)
7283 );
7284 assert_eq!(provider.resource_count(), 1);
7285 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7286 assert!(
7287 !harness
7288 .store
7289 .dump_text()
7290 .unwrap()
7291 .contains("provider-secret-needle")
7292 );
7293 let id = harness.store.attempts().unwrap()[0].id;
7294 assert_eq!(
7295 harness.store.attempt(id).unwrap().unwrap().outcome(),
7296 Some(&AttemptOutcome::failed(FailureReason::IsolationProvider(
7297 IsolationProviderFailure::RuntimeOperationFailed
7298 )))
7299 );
7300 let restarted = launcher_with_isolated(&harness, Arc::clone(&provider));
7301 restarted
7302 .recover_startup(std::slice::from_ref(&harness.policy))
7303 .await
7304 .expect("profile-local recovery");
7305 assert_eq!(
7306 provider.resource_count(),
7307 0,
7308 "the exact owned resource was destroyed once"
7309 );
7310 assert_eq!(
7311 harness.store.attempt(id).unwrap().unwrap().state(),
7312 AttemptState::Cleaned
7313 );
7314 let diagnostic =
7315 fs::read_to_string(restarted.diagnostics_root.join(format!("{id}.log"))).unwrap();
7316 assert!(diagnostic.contains("reason=isolation_runtime_operation_failed"));
7317 assert!(!diagnostic.contains("provider-secret-needle"));
7318 assert_eq!(
7319 provider
7320 .actions
7321 .lock()
7322 .unwrap()
7323 .iter()
7324 .filter(|action| **action == "destroy")
7325 .count(),
7326 1
7327 );
7328 assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7329 }
7330
7331 #[test]
7332 fn isolation_capability_and_adapter_failures_keep_closed_categories() {
7333 let cases = [
7334 (
7335 ProviderCapability::NotInstalled,
7336 IsolationProviderFailure::NotInstalled,
7337 ),
7338 (
7339 ProviderCapability::PermissionDenied,
7340 IsolationProviderFailure::PermissionDenied,
7341 ),
7342 (
7343 ProviderCapability::ImageUnavailableOrIncompatible,
7344 IsolationProviderFailure::ImageUnavailableOrIncompatible,
7345 ),
7346 (
7347 ProviderCapability::DiskQuotaUnavailable,
7348 IsolationProviderFailure::DiskQuotaUnavailable,
7349 ),
7350 (
7351 ProviderCapability::Degraded,
7352 IsolationProviderFailure::Degraded,
7353 ),
7354 ];
7355 for (capability, category) in cases {
7356 assert_eq!(
7357 capability.refusal(),
7358 FailureReason::IsolationProvider(category)
7359 );
7360 }
7361 let raw = FailureReason::Other("ghp_secret_from_provider".into());
7362 let safe = safe_isolation_reason(raw, IsolationProviderFailure::RuntimeOperationFailed);
7363 assert_eq!(
7364 safe,
7365 FailureReason::IsolationProvider(IsolationProviderFailure::RuntimeOperationFailed)
7366 );
7367 assert!(!safe.to_string().contains("ghp_secret"));
7368 assert_eq!(
7369 safe_isolation_reason(
7370 FailureReason::IsolationProvider(IsolationProviderFailure::PermissionDenied),
7371 IsolationProviderFailure::RuntimeOperationFailed,
7372 ),
7373 FailureReason::IsolationProvider(IsolationProviderFailure::PermissionDenied)
7374 );
7375 }
7376
7377 #[tokio::test]
7378 async fn isolated_permission_and_image_failures_stop_before_jit_without_native_fallback() {
7379 for image_failure in [false, true] {
7380 let harness = isolated_harness();
7381 let provider = Arc::new(FakeIsolatedProvider::default());
7382 provider
7383 .permission_denied
7384 .store(!image_failure, Ordering::SeqCst);
7385 provider
7386 .incompatible_image
7387 .store(image_failure, Ordering::SeqCst);
7388 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7389 launcher
7390 .recover_startup(std::slice::from_ref(&harness.policy))
7391 .await
7392 .unwrap();
7393 let error = harness
7394 .launch_through(&launcher)
7395 .await
7396 .expect_err("pre-JIT refusal");
7397 let expected = if image_failure {
7398 "resolved image is incompatible"
7399 } else {
7400 "permission denied"
7401 };
7402 assert!(error.reason.to_string().contains(expected));
7403 assert!(
7404 harness.store.attempts().unwrap().is_empty(),
7405 "no allocation was journalled"
7406 );
7407 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7408 assert_eq!(provider.handoffs.load(Ordering::SeqCst), 0);
7409 assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7410 }
7411 }
7412
7413 #[tokio::test]
7414 async fn unsupported_host_stops_before_jit_without_allocation_or_native_fallback() {
7415 let harness = isolated_harness();
7416 let provider = Arc::new(FakeIsolatedProvider::default());
7417 provider.unsupported_host.store(true, Ordering::SeqCst);
7418 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7419 launcher
7420 .recover_startup(std::slice::from_ref(&harness.policy))
7421 .await
7422 .unwrap();
7423 let error = harness
7424 .launch_through(&launcher)
7425 .await
7426 .expect_err("unsupported host must be refused before JIT");
7427
7428 assert_eq!(
7429 error.reason,
7430 FailureReason::IsolationProvider(IsolationProviderFailure::Unsupported)
7431 );
7432 assert!(harness.store.attempts().unwrap().is_empty());
7433 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
7434 assert_eq!(provider.handoffs.load(Ordering::SeqCst), 0);
7435 assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7436 assert!(provider.actions.lock().unwrap().is_empty());
7437 }
7438
7439 #[tokio::test]
7440 async fn no_journalled_environment_id_waits_for_provider_absence_proof() {
7441 let harness = isolated_harness();
7442 let provider = Arc::new(FakeIsolatedProvider::default());
7443 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7444 let runtime = harness.host_root().join("prepare-without-identity");
7445 fs::create_dir_all(&runtime).unwrap();
7446 let mut attempt = RunnerAttempt::allocate(
7447 AttemptId::from_u128(0x9191),
7448 harness.policy.id,
7449 runtime,
7450 harness.clock.now(),
7451 );
7452 let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7453 harness.policy.execution_policy()
7454 else {
7455 unreachable!()
7456 };
7457 attempt
7458 .allocate_execution(AttemptExecution::Isolated {
7459 provider_kind: Backend::Oci,
7460 environment_id: None,
7461 resolved_image: image.clone(),
7462 generation: "prepare-generation".into(),
7463 })
7464 .unwrap();
7465 attempt.begin_prepare(harness.clock.now()).unwrap();
7466 attempt
7467 .conclude(
7468 AttemptOutcome::failed(FailureReason::Other("prepare interrupted".into())),
7469 harness.clock.now(),
7470 )
7471 .unwrap();
7472 harness.store.record_attempt(&attempt).unwrap();
7473
7474 provider.recovery_unavailable.store(true, Ordering::SeqCst);
7475 launcher
7476 .clean(attempt.id)
7477 .await
7478 .expect_err("absence unproven");
7479 let deferred = harness.store.attempt(attempt.id).unwrap().unwrap();
7480 assert_eq!(deferred.state(), AttemptState::CleanupDeferred);
7481 assert!(deferred.counts_against_capacity());
7482
7483 provider.recovery_unavailable.store(false, Ordering::SeqCst);
7484 launcher.clean(attempt.id).await.expect("absence proven");
7485 assert_eq!(
7486 harness.store.attempt(attempt.id).unwrap().unwrap().state(),
7487 AttemptState::Cleaned
7488 );
7489 }
7490
7491 #[tokio::test]
7492 async fn isolated_terminal_capacity_waits_for_confirmed_absence_and_generation_matches() {
7493 let harness = isolated_harness();
7494 let provider = Arc::new(FakeIsolatedProvider::default());
7495 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7496 launcher
7497 .recover_startup(std::slice::from_ref(&harness.policy))
7498 .await
7499 .expect("ready");
7500 let mut attempt = harness.launch_through(&launcher).await.expect("launch");
7501 attempt
7502 .conclude(
7503 AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7504 harness.clock.now(),
7505 )
7506 .unwrap();
7507 harness.store.record_attempt(&attempt).unwrap();
7508 provider.destroy_deferred.store(true, Ordering::SeqCst);
7509 launcher
7510 .clean(attempt.id)
7511 .await
7512 .expect_err("provider deferred destroy");
7513 let deferred = harness.store.attempt(attempt.id).unwrap().unwrap();
7514 assert_eq!(deferred.state(), AttemptState::CleanupDeferred);
7515 assert!(deferred.counts_against_capacity());
7516 assert_eq!(provider.resource_count(), 1);
7517 provider.destroy_deferred.store(false, Ordering::SeqCst);
7518 let restarted = launcher_with_isolated(&harness, Arc::clone(&provider));
7519 restarted
7520 .recover_startup(std::slice::from_ref(&harness.policy))
7521 .await
7522 .expect("retry cleanup");
7523 assert_eq!(provider.resource_count(), 0);
7524 assert!(
7525 !harness
7526 .store
7527 .attempt(attempt.id)
7528 .unwrap()
7529 .unwrap()
7530 .counts_against_capacity()
7531 );
7532
7533 let second_id = AttemptId::from_u128(0x9999);
7535 let second_runtime = harness.host_root().join("generation-mismatch");
7536 fs::create_dir_all(&second_runtime).unwrap();
7537 let mut terminal = RunnerAttempt::allocate(
7538 second_id,
7539 harness.policy.id,
7540 second_runtime,
7541 harness.clock.now(),
7542 );
7543 let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7544 harness.policy.execution_policy()
7545 else {
7546 unreachable!()
7547 };
7548 terminal
7549 .allocate_execution(AttemptExecution::Isolated {
7550 provider_kind: Backend::Oci,
7551 environment_id: None,
7552 resolved_image: image.clone(),
7553 generation: "expected-generation".into(),
7554 })
7555 .unwrap();
7556 let mut stranger = FakeIsolatedProvider::identity(&terminal, harness.policy.host_id);
7557 let EnvironmentIdentity::Isolated { environment_id, .. } = &stranger else {
7558 unreachable!()
7559 };
7560 terminal.begin_prepare(harness.clock.now()).unwrap();
7561 terminal
7562 .prepared_environment(environment_id.clone())
7563 .unwrap();
7564 terminal.mark_prepared(harness.clock.now()).unwrap();
7565 terminal.jit_received(harness.clock.now()).unwrap();
7566 terminal.started_isolated(harness.clock.now()).unwrap();
7567 terminal
7568 .conclude(
7569 AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7570 harness.clock.now(),
7571 )
7572 .unwrap();
7573 harness.store.record_attempt(&terminal).unwrap();
7574 let EnvironmentIdentity::Isolated { generation, .. } = &mut stranger else {
7575 unreachable!()
7576 };
7577 *generation = "another-generation".into();
7578 provider
7579 .resources
7580 .lock()
7581 .unwrap()
7582 .insert(second_id, (stranger, EnvironmentState::Running));
7583 restarted
7584 .clean(terminal.id)
7585 .await
7586 .expect_err("ownership mismatch quarantined");
7587 assert_eq!(provider.resource_count(), 1);
7588 assert_eq!(
7589 harness.store.attempt(terminal.id).unwrap().unwrap().state(),
7590 AttemptState::CleanupDeferred
7591 );
7592 assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7593 }
7594
7595 #[tokio::test]
7596 async fn isolated_restart_at_each_transition_adopts_or_destroys_one_resource() {
7597 for stage in [
7598 AttemptState::Allocated,
7599 AttemptState::Preparing,
7600 AttemptState::Prepared,
7601 AttemptState::JitReceived,
7602 AttemptState::Starting,
7603 AttemptState::Idle,
7604 AttemptState::Busy,
7605 AttemptState::Finished,
7606 AttemptState::Failed,
7607 AttemptState::Orphaned,
7608 AttemptState::Destroying,
7609 AttemptState::CleanupDeferred,
7610 ] {
7611 let harness = isolated_harness();
7612 let provider = Arc::new(FakeIsolatedProvider::default());
7613 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7614 let id = AttemptId::from_u128(0x1000 + stage as u128);
7615 let runtime = harness.host_root().join(format!("phase-{stage}"));
7616 fs::create_dir_all(&runtime).unwrap();
7617 let mut attempt =
7618 RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
7619 let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7620 harness.policy.execution_policy()
7621 else {
7622 unreachable!()
7623 };
7624 attempt
7625 .allocate_execution(AttemptExecution::Isolated {
7626 provider_kind: Backend::Oci,
7627 environment_id: None,
7628 resolved_image: image.clone(),
7629 generation: format!("generation-{stage}"),
7630 })
7631 .unwrap();
7632 if stage != AttemptState::Allocated {
7633 attempt.begin_prepare(harness.clock.now()).unwrap();
7634 }
7635 let identity = FakeIsolatedProvider::identity(&attempt, harness.policy.host_id);
7636 if stage != AttemptState::Allocated {
7637 let resource_state = if matches!(
7638 stage,
7639 AttemptState::Finished
7640 | AttemptState::Failed
7641 | AttemptState::Orphaned
7642 | AttemptState::Destroying
7643 | AttemptState::CleanupDeferred
7644 ) {
7645 EnvironmentState::Exited
7646 } else {
7647 EnvironmentState::Running
7648 };
7649 provider
7650 .resources
7651 .lock()
7652 .unwrap()
7653 .insert(id, (identity.clone(), resource_state));
7654 }
7655 if !matches!(stage, AttemptState::Allocated | AttemptState::Preparing) {
7656 let EnvironmentIdentity::Isolated { environment_id, .. } = &identity else {
7657 unreachable!()
7658 };
7659 attempt
7660 .prepared_environment(environment_id.clone())
7661 .unwrap();
7662 attempt.mark_prepared(harness.clock.now()).unwrap();
7663 }
7664 if matches!(
7665 stage,
7666 AttemptState::JitReceived
7667 | AttemptState::Starting
7668 | AttemptState::Idle
7669 | AttemptState::Busy
7670 | AttemptState::Finished
7671 | AttemptState::Failed
7672 | AttemptState::Orphaned
7673 | AttemptState::Destroying
7674 | AttemptState::CleanupDeferred
7675 ) {
7676 attempt.jit_received(harness.clock.now()).unwrap();
7677 }
7678 if matches!(
7679 stage,
7680 AttemptState::Starting
7681 | AttemptState::Idle
7682 | AttemptState::Busy
7683 | AttemptState::Finished
7684 | AttemptState::Failed
7685 | AttemptState::Orphaned
7686 | AttemptState::Destroying
7687 | AttemptState::CleanupDeferred
7688 ) {
7689 attempt.started_isolated(harness.clock.now()).unwrap();
7690 }
7691 if matches!(
7692 stage,
7693 AttemptState::Idle | AttemptState::Busy | AttemptState::Finished
7694 ) {
7695 attempt.registered_idle(73, harness.clock.now()).unwrap();
7696 }
7697 if matches!(stage, AttemptState::Busy | AttemptState::Finished) {
7698 attempt.assigned_job(73, harness.clock.now()).unwrap();
7699 }
7700 if matches!(
7701 stage,
7702 AttemptState::Finished
7703 | AttemptState::Failed
7704 | AttemptState::Orphaned
7705 | AttemptState::Destroying
7706 | AttemptState::CleanupDeferred
7707 ) {
7708 let outcome = match stage {
7709 AttemptState::Finished => AttemptOutcome::CompletedJob,
7710 AttemptState::Orphaned => AttemptOutcome::Orphaned,
7711 _ => AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7712 };
7713 attempt.conclude(outcome, harness.clock.now()).unwrap();
7714 }
7715 if matches!(
7716 stage,
7717 AttemptState::Destroying | AttemptState::CleanupDeferred
7718 ) {
7719 attempt.begin_destroy(harness.clock.now()).unwrap();
7720 }
7721 if stage == AttemptState::CleanupDeferred {
7722 attempt.defer_cleanup(harness.clock.now()).unwrap();
7723 }
7724 harness.store.record_attempt(&attempt).unwrap();
7725 launcher
7726 .recover_startup(std::slice::from_ref(&harness.policy))
7727 .await
7728 .unwrap_or_else(|error| panic!("restart from {stage}: {error}"));
7729 let resumed = harness.store.attempt(id).unwrap().unwrap();
7730 if matches!(
7731 stage,
7732 AttemptState::Allocated
7733 | AttemptState::Preparing
7734 | AttemptState::Prepared
7735 | AttemptState::Finished
7736 | AttemptState::Failed
7737 | AttemptState::Orphaned
7738 | AttemptState::Destroying
7739 | AttemptState::CleanupDeferred
7740 ) {
7741 assert_eq!(
7742 resumed.state(),
7743 AttemptState::Cleaned,
7744 "{stage} must retire after restart"
7745 );
7746 } else {
7747 assert!(resumed.counts_against_capacity(), "live {stage} is adopted");
7748 assert_eq!(provider.resource_count(), 1);
7749 provider.resources.lock().unwrap().get_mut(&id).unwrap().1 =
7750 EnvironmentState::Exited;
7751 launcher.supervise(&harness.policy).await.unwrap();
7752 assert_eq!(
7753 harness.store.attempt(id).unwrap().unwrap().state(),
7754 AttemptState::Cleaned
7755 );
7756 }
7757 assert_eq!(
7758 provider.resource_count(),
7759 0,
7760 "{stage} resource cleaned exactly once"
7761 );
7762 let destroyed = provider
7763 .actions
7764 .lock()
7765 .unwrap()
7766 .iter()
7767 .filter(|action| **action == "destroy")
7768 .count();
7769 assert_eq!(
7770 destroyed,
7771 usize::from(stage != AttemptState::Allocated),
7772 "{stage} destroy count"
7773 );
7774 assert_eq!(
7775 harness.github.registrations.load(Ordering::SeqCst),
7776 0,
7777 "restart from {stage} cannot duplicate JIT"
7778 );
7779 assert_eq!(provider.native_calls.load(Ordering::SeqCst), 0);
7780 }
7781 }
7782
7783 #[tokio::test]
7784 async fn duplicate_provider_ownership_is_quarantined_until_unambiguous() {
7785 let harness = isolated_harness();
7786 let provider = Arc::new(FakeIsolatedProvider::default());
7787 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7788 let id = AttemptId::from_u128(0x7777);
7789 let runtime = harness.host_root().join("duplicate-provider-ownership");
7790 fs::create_dir_all(&runtime).unwrap();
7791 let mut attempt =
7792 RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
7793 let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7794 harness.policy.execution_policy()
7795 else {
7796 unreachable!()
7797 };
7798 attempt
7799 .allocate_execution(AttemptExecution::Isolated {
7800 provider_kind: Backend::Oci,
7801 environment_id: None,
7802 resolved_image: image.clone(),
7803 generation: "duplicate-generation".into(),
7804 })
7805 .unwrap();
7806 attempt.begin_prepare(harness.clock.now()).unwrap();
7807 harness.store.record_attempt(&attempt).unwrap();
7808 let first = FakeIsolatedProvider::identity(&attempt, harness.policy.host_id);
7809 let mut second = first.clone();
7810 let EnvironmentIdentity::Isolated { environment_id, .. } = &mut second else {
7811 unreachable!()
7812 };
7813 *environment_id = "another-environment".into();
7814 provider
7815 .resources
7816 .lock()
7817 .unwrap()
7818 .insert(id, (first, EnvironmentState::Running));
7819 provider.extra_resources.lock().unwrap().push(second);
7820 launcher
7821 .recover_startup(std::slice::from_ref(&harness.policy))
7822 .await
7823 .expect("duplicate is local");
7824 let deferred = harness.store.attempt(id).unwrap().unwrap();
7825 assert_eq!(deferred.state(), AttemptState::CleanupDeferred);
7826 assert!(deferred.counts_against_capacity());
7827 assert_eq!(provider.resource_count(), 1);
7828 assert_eq!(
7829 provider
7830 .actions
7831 .lock()
7832 .unwrap()
7833 .iter()
7834 .filter(|action| **action == "destroy")
7835 .count(),
7836 0
7837 );
7838 provider.extra_resources.lock().unwrap().clear();
7839 harness.clock.advance_secs(31);
7841 launcher
7842 .recover_startup(std::slice::from_ref(&harness.policy))
7843 .await
7844 .expect("single identity recovered");
7845 assert_eq!(provider.resource_count(), 0);
7846 assert_eq!(
7847 harness.store.attempt(id).unwrap().unwrap().state(),
7848 AttemptState::Cleaned
7849 );
7850 }
7851
7852 #[tokio::test]
7853 async fn provider_resource_without_live_journal_is_surfaced_but_never_deleted() {
7854 let harness = isolated_harness();
7855 let provider = Arc::new(FakeIsolatedProvider::default());
7856 let launcher = launcher_with_isolated(&harness, Arc::clone(&provider));
7857 let id = AttemptId::from_u128(0x8888);
7858 let runtime = harness.host_root().join("orphan-provider-resource");
7859 fs::create_dir_all(&runtime).unwrap();
7860 let mut attempt =
7861 RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
7862 let runner_manager_domain::execution::ExecutionPolicy::Isolated { image, .. } =
7863 harness.policy.execution_policy()
7864 else {
7865 unreachable!()
7866 };
7867 attempt
7868 .allocate_execution(AttemptExecution::Isolated {
7869 provider_kind: Backend::Oci,
7870 environment_id: None,
7871 resolved_image: image.clone(),
7872 generation: "orphan-generation".into(),
7873 })
7874 .unwrap();
7875 let identity = FakeIsolatedProvider::identity(&attempt, harness.policy.host_id);
7876 provider
7877 .resources
7878 .lock()
7879 .unwrap()
7880 .insert(id, (identity, EnvironmentState::Running));
7881 launcher
7882 .recover_startup(std::slice::from_ref(&harness.policy))
7883 .await
7884 .expect("orphan cannot block host");
7885 assert_eq!(provider.resource_count(), 1);
7886 assert!(
7887 harness
7888 .events
7889 .events()
7890 .contains(&AttemptEvent::QuarantinedOwnedResource { attempt: id })
7891 );
7892 assert_eq!(
7893 provider
7894 .actions
7895 .lock()
7896 .unwrap()
7897 .iter()
7898 .filter(|action| **action == "destroy")
7899 .count(),
7900 0
7901 );
7902
7903 let EnvironmentIdentity::Isolated { environment_id, .. } = provider
7904 .resources
7905 .lock()
7906 .unwrap()
7907 .get(&id)
7908 .unwrap()
7909 .0
7910 .clone()
7911 else {
7912 unreachable!()
7913 };
7914 attempt.begin_prepare(harness.clock.now()).unwrap();
7915 attempt.prepared_environment(environment_id).unwrap();
7916 attempt.mark_prepared(harness.clock.now()).unwrap();
7917 attempt.jit_received(harness.clock.now()).unwrap();
7918 attempt.started_isolated(harness.clock.now()).unwrap();
7919 attempt
7920 .conclude(
7921 AttemptOutcome::failed(FailureReason::ProcessStartFailed),
7922 harness.clock.now(),
7923 )
7924 .unwrap();
7925 attempt.begin_destroy(harness.clock.now()).unwrap();
7926 attempt.clean_isolated(harness.clock.now()).unwrap();
7927 harness.store.record_attempt(&attempt).unwrap();
7928 launcher
7929 .recover_startup(std::slice::from_ref(&harness.policy))
7930 .await
7931 .expect("cleaned row cannot authorize deletion");
7932 assert_eq!(provider.resource_count(), 1);
7933 assert_eq!(
7934 provider
7935 .actions
7936 .lock()
7937 .unwrap()
7938 .iter()
7939 .filter(|action| **action == "destroy")
7940 .count(),
7941 0
7942 );
7943 }
7944
7945 #[tokio::test]
7954 async fn a_root_that_refuses_a_launch_is_recorded_and_cleared_when_one_succeeds() {
7955 use runner_manager_platform::service::{clear_runner_root_refusal, runner_root_refusals};
7956
7957 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
7958 .with_host_runner_root();
7959 harness.ready().await;
7960
7961 let unusable = harness
7965 ._root
7966 .path()
7967 .join("absent")
7968 .join("deeper")
7969 .join("runners");
7970 let mut host = harness.host.clone();
7971 host.runner_root_override = Some(
7972 LocalAbsolutePath::new(unusable.to_str().expect("a UTF-8 temporary path"))
7973 .expect("a local absolute host root"),
7974 );
7975 harness.store.put_host(&host).unwrap();
7976
7977 let failure = harness
7978 .launch_result()
7979 .await
7980 .expect_err("a root whose parents are missing cannot hold a runner");
7981 assert!(
7982 matches!(failure.reason, FailureReason::Other(_)),
7983 "{failure:?}"
7984 );
7985
7986 let refusals = runner_root_refusals(&harness.app_paths).expect("readable");
7987 let refusal = refusals
7988 .first()
7989 .expect("the refusal reached the one surface that can hold it");
7990 assert_eq!(refusal.policy, harness.policy.id.to_string());
7991 assert_eq!(refusal.kind, "missing_parents");
7992 assert!(
7993 refusal.root.contains("runners") && refusal.detail.contains("runners"),
7994 "the directory must be named in full: {refusal:?}"
7995 );
7996
7997 harness.store.put_host(&harness.host).unwrap();
8000 harness.launch().await;
8001 assert!(
8002 runner_root_refusals(&harness.app_paths)
8003 .expect("readable")
8004 .is_empty(),
8005 "a successful placement clears that policy's record"
8006 );
8007
8008 clear_runner_root_refusal(&harness.app_paths, &harness.policy.id.to_string())
8009 .expect("cleanup");
8010 }
8011
8012 #[tokio::test]
8013 async fn a_launch_reports_how_long_each_step_took() {
8014 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8015 harness.ready().await;
8016 let started = harness.launch().await;
8017 let events = harness.events.events();
8018 let launched = events
8019 .iter()
8020 .position(|event| matches!(event, AttemptEvent::Launched { attempt, .. } if *attempt == started.id))
8021 .expect("a successful launch reports its step timings");
8022 let starting = events
8023 .iter()
8024 .position(|event| matches!(event, AttemptEvent::State { attempt, state: AttemptState::Starting } if *attempt == started.id))
8025 .unwrap();
8026 assert!(starting < launched, "timings follow the start: {events:?}");
8027 }
8028
8029 #[tokio::test]
8030 async fn a_failed_launch_still_reports_its_step_timings() {
8031 let harness = Harness::new(
8032 FakeGithubLifecycle::default().fail(true),
8033 Arc::new(PersistentDemand),
8034 );
8035 harness.ready().await;
8036 let failed = harness.launch_after_waiting(Duration::from_secs(90)).await;
8037 assert!(failed.is_err());
8038
8039 let timings = harness
8040 .events
8041 .events()
8042 .into_iter()
8043 .find_map(|event| match event {
8044 AttemptEvent::Launched {
8045 started: false,
8046 timings,
8047 ..
8048 } => Some(timings),
8049 _ => None,
8050 })
8051 .expect("a launch that failed at registration reports its timings");
8052 assert_eq!(timings.lock_wait, Duration::from_secs(90));
8053 assert_eq!(timings.order_wait, Duration::from_secs(2));
8054 assert_eq!(timings.spawn, Duration::ZERO, "spawn was never reached");
8055 assert!(timings.total() >= Duration::from_secs(92));
8056 }
8057
8058 #[cfg(target_os = "macos")]
8059 #[tokio::test]
8060 async fn a_launch_removes_the_seed_of_a_root_no_longer_in_use() {
8061 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8062 harness.ready().await;
8063
8064 let old_root = harness._root.path().join("old-root");
8066 let old_runtime = old_root.join("abandoned");
8067 fs::create_dir_all(old_root.join(PACKAGE_SEED_DIR).join("2.335.0")).unwrap();
8068 fs::create_dir_all(&old_runtime).unwrap();
8069 let old = RunnerAttempt::allocate(
8070 AttemptId::new_random(),
8071 harness.policy.id,
8072 &old_runtime,
8073 harness.clock.now(),
8074 );
8075 harness.store.record_attempt(&old).unwrap();
8076 let kept = harness.host_root().join(PACKAGE_SEED_DIR);
8077 fs::create_dir_all(kept.join("2.336.0")).unwrap();
8078
8079 harness.launch().await;
8080
8081 assert!(
8082 !old_root.join(PACKAGE_SEED_DIR).exists(),
8083 "the abandoned root's seed is removed"
8084 );
8085 assert!(old_runtime.exists(), "only the seed is touched");
8086 assert!(kept.exists(), "the host root's own seed is kept");
8087 }
8088
8089 #[tokio::test]
8090 async fn a_job_walks_every_state_and_cleans_every_artifact() {
8091 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8092 harness.ready().await;
8093 let started = harness.launch().await;
8094 assert_eq!(started.state(), AttemptState::Starting);
8095 assert_eq!(read_runner_id(started.runtime_path()), Some(73));
8096
8097 harness
8098 .github
8099 .observe(GithubRunnerObservation::Registered { busy: false });
8100 harness.launcher.supervise(&harness.policy).await.unwrap();
8101 assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8102
8103 harness
8104 .github
8105 .observe(GithubRunnerObservation::Registered { busy: true });
8106 harness.launcher.supervise(&harness.policy).await.unwrap();
8107 assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8108
8109 harness.processes.finish_successfully();
8110 harness
8111 .github
8112 .observe(GithubRunnerObservation::NotRegistered);
8113 harness.launcher.supervise(&harness.policy).await.unwrap();
8114 let cleaned = harness.only_attempt();
8115 assert_eq!(cleaned.state(), AttemptState::Cleaned);
8116 assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
8117 assert!(!started.runtime_path().exists());
8118 assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8119 assert_eq!(harness.github.remaining_runners.load(Ordering::SeqCst), 0);
8120
8121 let states: Vec<_> = harness
8122 .events
8123 .events()
8124 .into_iter()
8125 .filter_map(|event| match event {
8126 AttemptEvent::State { state, .. } => Some(state),
8127 _ => None,
8128 })
8129 .collect();
8130 assert_eq!(
8131 states,
8132 vec![
8133 AttemptState::Allocated,
8134 AttemptState::JitReceived,
8135 AttemptState::Starting,
8136 AttemptState::Idle,
8137 AttemptState::Busy,
8138 AttemptState::Finished,
8139 AttemptState::Cleaned,
8140 ]
8141 );
8142 }
8143
8144 #[tokio::test]
8145 async fn a_cleaned_ephemeral_attempt_reaps_a_directory_recreated_after_cleanup() {
8146 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8147 harness.ready().await;
8148 let started = harness.launch().await;
8149 let runtime = started.runtime_path().to_path_buf();
8150
8151 harness
8152 .github
8153 .observe(GithubRunnerObservation::Registered { busy: true });
8154 harness.launcher.supervise(&harness.policy).await.unwrap();
8155 harness.processes.finish_successfully();
8156 harness
8157 .github
8158 .observe(GithubRunnerObservation::NotRegistered);
8159 harness.launcher.supervise(&harness.policy).await.unwrap();
8160
8161 assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8162 assert!(!runtime.exists());
8163 assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8164
8165 let residue = runtime.join("_work").join("late-node-process");
8169 fs::create_dir_all(&residue).unwrap();
8170 fs::write(residue.join("node_modules.lock"), b"late residue").unwrap();
8171
8172 harness.launcher.supervise(&harness.policy).await.unwrap();
8173
8174 assert!(!runtime.exists(), "late ephemeral residue is reaped");
8175 assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8176 assert_eq!(
8177 harness.packages.releases.load(Ordering::SeqCst),
8178 1,
8179 "reaping residue does not release the package lease twice"
8180 );
8181 }
8182
8183 #[tokio::test]
8184 async fn locked_late_residue_never_blocks_startup_recovery() {
8185 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8186 harness.ready().await;
8187 let started = harness.launch().await;
8188 let runtime = started.runtime_path().to_path_buf();
8189
8190 harness
8191 .github
8192 .observe(GithubRunnerObservation::Registered { busy: true });
8193 harness.launcher.supervise(&harness.policy).await.unwrap();
8194 harness.processes.finish_successfully();
8195 harness
8196 .github
8197 .observe(GithubRunnerObservation::NotRegistered);
8198 harness.launcher.supervise(&harness.policy).await.unwrap();
8199 assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8200
8201 let held = runtime.join("_work").join("late-node-process");
8202 fs::create_dir_all(&held).unwrap();
8203 fs::write(held.join("node_modules.lock"), b"late residue").unwrap();
8204 let Some(block) = BlockedDeletion::inject(&held) else {
8205 eprintln!(
8206 "skipped: this account cannot be refused a deletion, so locked late residue cannot be injected"
8207 );
8208 return;
8209 };
8210
8211 let restarted = harness.restart();
8212 restarted
8213 .recover_startup(std::slice::from_ref(&harness.policy))
8214 .await
8215 .expect("late residue cannot take the daemon offline");
8216 assert!(runtime.exists(), "the locked residue remains for a retry");
8217 assert!(
8218 harness
8219 .reconcile_events
8220 .events()
8221 .iter()
8222 .any(|event| matches!(
8223 event,
8224 LifecycleEvent::AttemptCleanFailed {
8225 attempt,
8226 reason: "late_ephemeral_workspace_could_not_be_removed",
8227 ..
8228 } if *attempt == started.id
8229 )),
8230 "the non-blocking cleanup failure remains visible"
8231 );
8232
8233 block.release();
8234 harness.clock.advance_secs(31);
8236 restarted
8237 .supervise(&harness.policy)
8238 .await
8239 .expect("the ordinary retry succeeds after the lock is released");
8240 assert!(!runtime.exists(), "the retry removes the late residue");
8241 assert_eq!(harness.attempt(started.id).state(), AttemptState::Cleaned);
8242 assert_eq!(
8243 harness.packages.releases.load(Ordering::SeqCst),
8244 1,
8245 "retrying residue never releases the package lease twice"
8246 );
8247 }
8248
8249 #[tokio::test]
8250 async fn mixed_locked_cleanup_residue_isolated_from_the_whole_startup_pass() {
8251 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8252 harness.ready().await;
8253
8254 let terminal = harness.launch().await;
8258 let terminal_runtime = terminal.runtime_path().to_path_buf();
8259 harness.conclude(terminal.id);
8260 let Some(terminal_block) = BlockedDeletion::inject(&terminal_runtime.join("late-child"))
8261 else {
8262 eprintln!("skipped: this account cannot inject the two independent deletion refusals");
8263 return;
8264 };
8265
8266 let cleaned = harness.launch().await;
8270 let cleaned_runtime = cleaned.runtime_path().to_path_buf();
8271 harness.conclude(cleaned.id);
8272 harness
8273 .launcher
8274 .clean(cleaned.id)
8275 .await
8276 .expect("the second attempt initially cleans");
8277 let Some(cleaned_block) =
8278 BlockedDeletion::inject(&cleaned_runtime.join("_work").join("late-child"))
8279 else {
8280 terminal_block.release();
8281 eprintln!("skipped: this account cannot inject the two independent deletion refusals");
8282 return;
8283 };
8284
8285 let restarted = harness.restart();
8286 for _ in 0..2 {
8287 restarted
8288 .recover_startup(std::slice::from_ref(&harness.policy))
8289 .await
8290 .expect("repeated recovery remains ready while both residues are locked");
8291 }
8292 assert_ne!(
8293 harness.attempt(terminal.id).state(),
8294 AttemptState::Cleaned,
8295 "the journal does not claim the terminal workspace was removed"
8296 );
8297 assert_eq!(
8298 harness.attempt(cleaned.id).state(),
8299 AttemptState::Cleaned,
8300 "late residue does not undo an already durable cleaned transition"
8301 );
8302 assert!(terminal_runtime.exists());
8303 assert!(cleaned_runtime.exists());
8304 assert_eq!(
8305 harness.packages.releases.load(Ordering::SeqCst),
8306 1,
8307 "repeated recovery neither loses nor duplicates package leases"
8308 );
8309
8310 terminal_block.release();
8311 cleaned_block.release();
8312 harness.clock.advance_secs(31);
8314 restarted
8315 .supervise(&harness.policy)
8316 .await
8317 .expect("one ordinary pass clears both residues after their locks disappear");
8318 assert!(!terminal_runtime.exists());
8319 assert!(!cleaned_runtime.exists());
8320 assert_eq!(harness.attempt(terminal.id).state(), AttemptState::Cleaned);
8321 assert_eq!(harness.attempt(cleaned.id).state(), AttemptState::Cleaned);
8322 assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 2);
8323 }
8324
8325 #[tokio::test]
8326 async fn a_terminal_attempt_whose_runtime_is_already_gone_recovers_cleanly() {
8327 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8328 harness.ready().await;
8329 let attempt = harness.launch().await;
8330 let runtime = attempt.runtime_path().to_path_buf();
8331 harness.conclude(attempt.id);
8332 fs::remove_dir_all(&runtime).unwrap();
8333
8334 harness
8335 .restart()
8336 .recover_startup(std::slice::from_ref(&harness.policy))
8337 .await
8338 .expect("an already absent disposable runtime is successful cleanup");
8339
8340 assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
8341 assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8342 }
8343
8344 #[tokio::test]
8345 async fn a_non_workspace_cleanup_failure_still_fails_closed_and_can_retry() {
8346 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8347 harness.ready().await;
8348 let attempt = harness.launch().await;
8349 let runtime = attempt.runtime_path().to_path_buf();
8350 harness.conclude(attempt.id);
8351 harness.packages.fail_releases(1);
8352
8353 let restarted = harness.restart();
8354 let failure = restarted
8355 .recover_startup(std::slice::from_ref(&harness.policy))
8356 .await
8357 .expect_err("a package-accounting failure is not safe to downgrade to residue");
8358 assert!(
8359 failure
8360 .reason()
8361 .to_string()
8362 .contains("package lease could not be released")
8363 );
8364 assert!(
8365 !runtime.exists(),
8366 "workspace removal completed before release failed"
8367 );
8368 assert_ne!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
8369
8370 restarted
8371 .recover_startup(std::slice::from_ref(&harness.policy))
8372 .await
8373 .expect("the same journal entry retries safely after the transient failure");
8374 assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
8375 assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8376 }
8377
8378 #[tokio::test]
8379 async fn startup_never_scrubs_a_cleaned_persistent_work_directory() {
8380 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
8381 .with_persistent_workspace(1);
8382 harness.ready().await;
8383 let attempt = harness.launch().await;
8384 let retained = attempt.runtime_path().join("_work").join("checkout-marker");
8385 fs::create_dir_all(retained.parent().unwrap()).unwrap();
8386 fs::write(&retained, b"persistent checkout").unwrap();
8387 harness.cleanup_retaining_work(attempt.id).await;
8388
8389 harness
8390 .restart()
8391 .recover_startup(std::slice::from_ref(&harness.policy))
8392 .await
8393 .expect("a cleaned persistent slot is already reconciled");
8394
8395 assert_eq!(fs::read(&retained).unwrap(), b"persistent checkout");
8396 assert_eq!(harness.packages.releases.load(Ordering::SeqCst), 1);
8397 }
8398
8399 #[tokio::test]
8400 async fn an_idle_exit_is_not_a_failure_in_the_journal_or_events() {
8401 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8402 harness.ready().await;
8403 let started = harness.launch().await;
8404 harness
8405 .github
8406 .observe(GithubRunnerObservation::Registered { busy: false });
8407 harness.launcher.supervise(&harness.policy).await.unwrap();
8408 harness.clock.advance_secs(11);
8409 harness.processes.set_alive(false);
8410 harness
8411 .github
8412 .observe(GithubRunnerObservation::NotRegistered);
8413 harness.launcher.supervise(&harness.policy).await.unwrap();
8414
8415 let cleaned = harness.only_attempt();
8416 assert!(cleaned.outcome().unwrap().is_idle_exit());
8417 assert!(!cleaned.outcome().unwrap().is_failure());
8418 assert!(!started.runtime_path().exists());
8419 assert!(
8420 harness
8421 .reconcile_events
8422 .events()
8423 .iter()
8424 .any(|event| matches!(
8425 event,
8426 LifecycleEvent::AttemptCleaned {
8427 outcome: OutcomeKind::IdleExit,
8428 ..
8429 }
8430 ))
8431 );
8432 assert!(!harness.events.events().iter().any(|event| matches!(
8433 event,
8434 AttemptEvent::Concluded {
8435 outcome: OutcomeKind::Failed,
8436 ..
8437 }
8438 )));
8439 }
8440
8441 #[tokio::test]
8442 async fn handoff_is_absent_after_success_and_every_failed_spawn_retry() {
8443 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8444 harness.processes.fail_spawns(2);
8445 harness.ready().await;
8446 let attempt = harness.launch().await;
8447 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 3);
8448 assert!(harness.processes.saw_secret.load(Ordering::SeqCst));
8449 let names: Vec<_> = fs::read_dir(attempt.runtime_path())
8450 .unwrap()
8451 .map(|entry| entry.unwrap().file_name())
8452 .collect();
8453 assert!(
8454 names.iter().all(|name| {
8455 !name
8456 .to_string_lossy()
8457 .starts_with(RestrictiveHandoff::NAME_PREFIX)
8458 }),
8459 "JIT artifact survived: {names:?}"
8460 );
8461 assert_eq!(
8462 *harness.delay.0.lock().unwrap(),
8463 vec![Duration::from_millis(10), Duration::from_millis(20)]
8464 );
8465 }
8466
8467 #[tokio::test]
8468 async fn jit_retry_stops_with_demand_and_a_terminal_403_never_retries() {
8469 let gone = Harness::new(
8470 FakeGithubLifecycle::default().fail(false),
8471 Arc::new(FakeDemand::answering([false])),
8472 );
8473 gone.ready().await;
8474 assert!(gone.launch_result().await.is_err());
8475 assert_eq!(gone.github.registrations.load(Ordering::SeqCst), 1);
8476 assert!(gone.delay.0.lock().unwrap().is_empty());
8477
8478 let forbidden = Harness::new(
8479 FakeGithubLifecycle::default().fail(true),
8480 Arc::new(PersistentDemand),
8481 );
8482 forbidden.ready().await;
8483 assert!(forbidden.launch_result().await.is_err());
8484 assert_eq!(forbidden.github.registrations.load(Ordering::SeqCst), 1);
8485 assert!(forbidden.delay.0.lock().unwrap().is_empty());
8486 assert!(matches!(
8487 forbidden.only_attempt().outcome(),
8488 Some(AttemptOutcome::Failed {
8489 reason: FailureReason::Other(action)
8490 }) if action.contains("403")
8491 ));
8492
8493 let transient = Harness::new(
8494 FakeGithubLifecycle::default().fail(false).fail(false),
8495 Arc::new(PersistentDemand),
8496 );
8497 transient.ready().await;
8498 transient.launch().await;
8499 assert_eq!(transient.github.registrations.load(Ordering::SeqCst), 3);
8500 assert_eq!(
8501 *transient.delay.0.lock().unwrap(),
8502 vec![Duration::from_millis(10), Duration::from_millis(20)]
8503 );
8504 }
8505
8506 #[test]
8514 fn a_workspace_leaves_room_for_the_deepest_path_a_checkout_writes() {
8515 const MAX_PATH: usize = 260;
8516 let root = r"C:\Users\IvanD\AppData\Local\IvanMurzak\runner-manager\data\runtime";
8518 let repo = "GitHub-Runner-Scaler-UI";
8522 let deepest = format!(
8523 r"_work\{repo}\{repo}\.git\objects\pack\pack-{}.keep",
8524 "0".repeat(40)
8525 );
8526
8527 let name = workspace_name(AttemptId::new_random());
8528 assert_eq!(name.len(), WORKSPACE_NAME_LEN, "{name}");
8529 assert!(
8530 name.chars().all(|c| c.is_ascii_hexdigit()),
8531 "a directory name must not carry the identifier's dashes: {name}"
8532 );
8533
8534 let full = format!(r"{root}\{name}\{deepest}");
8535 assert!(
8536 full.len() < MAX_PATH,
8537 "the deepest path a checkout writes must fit: {} characters, limit {MAX_PATH}",
8538 full.len()
8539 );
8540
8541 let old = format!(
8544 r"{root}\{}\{}\{deepest}",
8545 PolicyId::new_random(),
8546 AttemptId::new_random()
8547 );
8548 assert!(
8549 old.len() > MAX_PATH,
8550 "the old layout is supposed to be the thing that did not fit: {} characters",
8551 old.len()
8552 );
8553 }
8554
8555 #[tokio::test]
8556 async fn two_attempts_never_share_a_workspace_even_after_failure() {
8557 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8558 harness.ready().await;
8559 let first = harness.launch().await;
8560 fs::write(first.runtime_path().join("hostile-leftover"), b"first job").unwrap();
8561 harness
8562 .github
8563 .observe(GithubRunnerObservation::Registered { busy: false });
8564 harness.launcher.supervise(&harness.policy).await.unwrap();
8565 harness.clock.advance_secs(11);
8566 harness.processes.set_alive(false);
8567 harness
8568 .github
8569 .observe(GithubRunnerObservation::NotRegistered);
8570 harness.launcher.supervise(&harness.policy).await.unwrap();
8571 assert!(!first.runtime_path().exists());
8572
8573 let second = harness.launch().await;
8574 assert_ne!(first.runtime_path(), second.runtime_path());
8575 assert!(!second.runtime_path().join("hostile-leftover").exists());
8576
8577 fs::write(
8578 second.runtime_path().join("hostile-on-failure"),
8579 b"second job",
8580 )
8581 .unwrap();
8582 harness.processes.set_alive(false);
8583 harness
8584 .github
8585 .observe(GithubRunnerObservation::NotRegistered);
8586 harness.launcher.supervise(&harness.policy).await.unwrap();
8587 assert!(
8588 !second.runtime_path().exists(),
8589 "failed workspace was retained"
8590 );
8591 }
8592
8593 #[tokio::test]
8594 async fn a_runner_that_never_gets_a_job_is_stopped_deregistered_and_not_replaced() {
8595 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8596 harness.ready().await;
8597 let attempt = harness.launch().await;
8598
8599 harness
8603 .github
8604 .observe(GithubRunnerObservation::Registered { busy: false });
8605 harness.launcher.supervise(&harness.policy).await.unwrap();
8606 assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8607
8608 harness.clock.advance_secs(9);
8611 harness
8612 .github
8613 .observe(GithubRunnerObservation::Registered { busy: false });
8614 let none_yet = harness.launcher.supervise(&harness.policy).await.unwrap();
8615 assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8616 assert!(none_yet.is_empty());
8617 assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 0);
8618
8619 harness.clock.advance_secs(1);
8621 harness
8622 .github
8623 .observe(GithubRunnerObservation::Registered { busy: false });
8624 let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8625
8626 let concluded = harness.store.attempt(attempt.id).unwrap().unwrap();
8627 assert_eq!(
8628 concluded.outcome(),
8629 Some(&AttemptOutcome::ExitedIdleWithoutWork),
8630 "a surplus runner did not fail; recording one as a failure sends an operator \
8631 hunting a fault that does not exist"
8632 );
8633 assert_eq!(concluded.state(), AttemptState::Cleaned);
8634 assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 1);
8635 assert!(!attempt.runtime_path().exists());
8636
8637 assert_eq!(
8640 *harness.github.deregistrations.lock().unwrap(),
8641 vec![73],
8642 "the attempt's own runner id, deleted exactly once"
8643 );
8644
8645 assert!(
8648 replacements.is_empty(),
8649 "a surplus exit must not request a replacement"
8650 );
8651 }
8652
8653 #[tokio::test]
8654 async fn a_registration_github_will_not_delete_still_concludes_the_attempt() {
8655 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8659 harness.ready().await;
8660 let attempt = harness.launch().await;
8661 harness
8662 .github
8663 .observe(GithubRunnerObservation::Registered { busy: false });
8664 harness.launcher.supervise(&harness.policy).await.unwrap();
8665
8666 harness
8667 .github
8668 .deregistration_fails
8669 .store(true, Ordering::SeqCst);
8670 harness.clock.advance_secs(11);
8671 harness
8672 .github
8673 .observe(GithubRunnerObservation::Registered { busy: false });
8674 harness.launcher.supervise(&harness.policy).await.unwrap();
8675
8676 assert_eq!(
8677 *harness.github.deregistrations.lock().unwrap(),
8678 vec![73],
8679 "the delete was attempted"
8680 );
8681 let concluded = harness.store.attempt(attempt.id).unwrap().unwrap();
8682 assert_eq!(
8683 concluded.outcome(),
8684 Some(&AttemptOutcome::ExitedIdleWithoutWork),
8685 "the attempt concluded anyway"
8686 );
8687 assert_eq!(concluded.state(), AttemptState::Cleaned);
8688 assert!(!attempt.runtime_path().exists());
8689 }
8690
8691 #[tokio::test]
8692 async fn exit_before_acceptance_returns_replacement_intent_without_launching() {
8693 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8694 harness.ready().await;
8695 let first = harness.launch().await;
8696 harness.processes.set_alive(false);
8697 harness
8698 .github
8699 .observe(GithubRunnerObservation::NotRegistered);
8700 let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8701 let failed = harness.store.attempt(first.id).unwrap().unwrap();
8702 assert!(matches!(
8703 failed.outcome(),
8704 Some(AttemptOutcome::Failed {
8705 reason: FailureReason::ProcessExitedUnexpectedly
8706 })
8707 ));
8708 assert!(!first.runtime_path().exists());
8709
8710 assert_eq!(
8711 replacements,
8712 vec![ReplacementIntent {
8713 policy: harness.policy.id,
8714 previous_attempt: first.id,
8715 operation: "exit_before_acceptance_replacement",
8716 }]
8717 );
8718 assert_eq!(harness.store.attempts().unwrap().len(), 1);
8719 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 1);
8720 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 1);
8721 assert!(harness.delay.0.lock().unwrap().is_empty());
8722 }
8723
8724 fn write_runner_diagnostics(runtime: &Path, ran_a_job: bool) {
8728 let diag = runtime.join("_diag");
8729 fs::create_dir_all(&diag).unwrap();
8730 fs::write(diag.join("Runner_20261007-211941-utc.log"), b"listener").unwrap();
8731 if ran_a_job {
8732 fs::write(diag.join("Worker_20261007-211950-utc.log"), b"worker").unwrap();
8733 }
8734 }
8735
8736 #[tokio::test]
8741 async fn a_job_too_short_to_be_seen_busy_is_recorded_as_completed() {
8742 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8743 harness.ready().await;
8744 let started = harness.launch().await;
8745 assert_eq!(started.state(), AttemptState::Starting);
8746 write_runner_diagnostics(started.runtime_path(), true);
8747 harness.processes.finish_successfully();
8748 harness
8749 .github
8750 .observe(GithubRunnerObservation::NotRegistered);
8751
8752 let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8753
8754 let cleaned = harness.only_attempt();
8755 assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
8756 assert_eq!(cleaned.state(), AttemptState::Cleaned);
8757 assert_eq!(cleaned.github_runner_id(), Some(73));
8758 assert!(
8759 replacements.is_empty(),
8760 "a finished job needs no replacement"
8761 );
8762 let busy_seen = harness.events.events().into_iter().any(|event| {
8763 matches!(
8764 event,
8765 AttemptEvent::State {
8766 state: AttemptState::Busy,
8767 ..
8768 }
8769 )
8770 });
8771 assert!(busy_seen, "the journal walks the diagram to `busy` first");
8772 }
8773
8774 #[tokio::test]
8776 async fn a_job_taken_between_two_passes_after_idle_is_recorded_as_completed() {
8777 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8778 harness.ready().await;
8779 let started = harness.launch().await;
8780 harness
8781 .github
8782 .observe(GithubRunnerObservation::Registered { busy: false });
8783 harness.launcher.supervise(&harness.policy).await.unwrap();
8784 assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8785
8786 write_runner_diagnostics(started.runtime_path(), true);
8787 harness.processes.finish_successfully();
8788 harness
8789 .github
8790 .observe(GithubRunnerObservation::NotRegistered);
8791 let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
8792
8793 assert_eq!(
8794 harness.only_attempt().outcome(),
8795 Some(&AttemptOutcome::CompletedJob)
8796 );
8797 assert!(replacements.is_empty());
8798 }
8799
8800 #[tokio::test]
8804 async fn an_assignment_the_runner_never_took_up_is_not_a_completed_job() {
8805 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8806 harness.ready().await;
8807 let started = harness.launch().await;
8808 harness
8809 .github
8810 .observe(GithubRunnerObservation::Registered { busy: true });
8811 harness.launcher.supervise(&harness.policy).await.unwrap();
8812 assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8813
8814 write_runner_diagnostics(started.runtime_path(), false);
8815 harness.processes.finish_successfully();
8816 harness
8817 .github
8818 .observe(GithubRunnerObservation::NotRegistered);
8819 harness.launcher.supervise(&harness.policy).await.unwrap();
8820
8821 let cleaned = harness.only_attempt();
8822 assert_eq!(
8823 cleaned.outcome(),
8824 Some(&AttemptOutcome::ExitedIdleWithoutWork)
8825 );
8826 assert_eq!(cleaned.state(), AttemptState::Cleaned);
8827 }
8828
8829 #[tokio::test]
8832 async fn a_finished_job_still_listed_at_github_is_completed_and_deregistered() {
8833 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8834 harness.ready().await;
8835 let started = harness.launch().await;
8836 harness
8837 .github
8838 .observe(GithubRunnerObservation::Registered { busy: true });
8839 harness.launcher.supervise(&harness.policy).await.unwrap();
8840
8841 write_runner_diagnostics(started.runtime_path(), true);
8842 harness.processes.finish_successfully();
8843 harness
8844 .github
8845 .observe(GithubRunnerObservation::Registered { busy: true });
8846 harness.launcher.supervise(&harness.policy).await.unwrap();
8847
8848 assert_eq!(
8849 harness.only_attempt().outcome(),
8850 Some(&AttemptOutcome::CompletedJob)
8851 );
8852 assert_eq!(*harness.github.deregistrations.lock().unwrap(), vec![73]);
8853 }
8854
8855 #[tokio::test]
8858 async fn a_live_runner_whose_assignment_was_withdrawn_goes_back_to_idle() {
8859 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8860 harness.ready().await;
8861 let started = harness.launch().await;
8862 harness
8863 .github
8864 .observe(GithubRunnerObservation::Registered { busy: true });
8865 harness.launcher.supervise(&harness.policy).await.unwrap();
8866 assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8867
8868 write_runner_diagnostics(started.runtime_path(), false);
8869 harness
8870 .github
8871 .observe(GithubRunnerObservation::Registered { busy: false });
8872 harness.launcher.supervise(&harness.policy).await.unwrap();
8873 assert_eq!(harness.only_attempt().state(), AttemptState::Idle);
8874
8875 harness
8877 .github
8878 .observe(GithubRunnerObservation::Registered { busy: true });
8879 harness.launcher.supervise(&harness.policy).await.unwrap();
8880 write_runner_diagnostics(started.runtime_path(), true);
8881 harness
8882 .github
8883 .observe(GithubRunnerObservation::Registered { busy: false });
8884 harness.launcher.supervise(&harness.policy).await.unwrap();
8885 assert_eq!(harness.only_attempt().state(), AttemptState::Busy);
8886 }
8887
8888 #[tokio::test]
8891 async fn a_clean_exit_waits_for_github_to_answer() {
8892 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8893 harness.ready().await;
8894 let started = harness.launch().await;
8895 write_runner_diagnostics(started.runtime_path(), true);
8896 harness.processes.finish_successfully();
8897 harness.github.observe(GithubRunnerObservation::Unreachable);
8898 harness.launcher.supervise(&harness.policy).await.unwrap();
8899 assert_eq!(harness.only_attempt().state(), AttemptState::Starting);
8900
8901 harness
8902 .github
8903 .observe(GithubRunnerObservation::NotRegistered);
8904 harness.launcher.supervise(&harness.policy).await.unwrap();
8905 assert_eq!(
8906 harness.only_attempt().outcome(),
8907 Some(&AttemptOutcome::CompletedJob)
8908 );
8909 }
8910
8911 #[tokio::test]
8914 async fn a_short_job_with_no_runner_id_still_concludes() {
8915 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8916 harness.ready().await;
8917 let started = harness.launch().await;
8918 fs::remove_file(started.runtime_path().join(RUNNER_ID_FILE)).unwrap();
8919 write_runner_diagnostics(started.runtime_path(), true);
8920 harness.processes.finish_successfully();
8921 harness
8922 .github
8923 .observe(GithubRunnerObservation::NotRegistered);
8924 harness
8925 .launcher
8926 .supervise(&harness.policy)
8927 .await
8928 .expect("supervision goes on");
8929 assert!(harness.only_attempt().is_terminal());
8930 }
8931
8932 #[tokio::test]
8935 async fn without_runner_diagnostics_an_early_exit_is_still_a_failure() {
8936 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
8937 harness.ready().await;
8938 harness.launch().await;
8939 harness.processes.finish_successfully();
8940 harness
8941 .github
8942 .observe(GithubRunnerObservation::NotRegistered);
8943 harness.launcher.supervise(&harness.policy).await.unwrap();
8944 assert!(matches!(
8945 harness.only_attempt().outcome(),
8946 Some(AttemptOutcome::Failed {
8947 reason: FailureReason::ProcessExitedUnexpectedly
8948 })
8949 ));
8950 }
8951
8952 #[tokio::test]
8953 async fn expired_jit_is_removed_and_does_not_reregister_after_demand_disappears() {
8954 let harness = Harness::new(
8955 FakeGithubLifecycle::default(),
8956 Arc::new(FakeDemand::answering([false])),
8957 );
8958 let id = AttemptId::new_random();
8959 let runtime = harness
8960 .launcher
8961 .app_paths
8962 .runtime_dir()
8963 .join(harness.policy.id.to_string())
8964 .join(id.to_string());
8965 fs::create_dir_all(&runtime).unwrap();
8966 let mut attempt =
8967 RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
8968 attempt.jit_received(harness.clock.now()).unwrap();
8969 harness.store.record_attempt(&attempt).unwrap();
8970 harness.clock.advance_secs(11);
8971 let replacements = harness
8972 .launcher
8973 .recover_startup(std::slice::from_ref(&harness.policy))
8974 .await
8975 .unwrap();
8976 assert_eq!(
8977 replacements,
8978 vec![ReplacementIntent {
8979 policy: harness.policy.id,
8980 previous_attempt: id,
8981 operation: "jit_expired_replacement",
8982 }]
8983 );
8984
8985 let cleaned = harness.store.attempt(id).unwrap().unwrap();
8986 assert_eq!(cleaned.state(), AttemptState::Cleaned);
8987 assert!(matches!(
8988 cleaned.outcome(),
8989 Some(AttemptOutcome::Failed {
8990 reason: FailureReason::JitExpired
8991 })
8992 ));
8993 assert!(!runtime.exists());
8994 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
8995 assert!(harness.delay.0.lock().unwrap().is_empty());
8996 }
8997
8998 #[tokio::test]
8999 async fn expired_jit_returns_intent_but_never_launches_inside_lifecycle() {
9000 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9001 let id = AttemptId::new_random();
9002 let runtime = harness
9003 .launcher
9004 .app_paths
9005 .runtime_dir()
9006 .join("expired-with-demand");
9007 fs::create_dir_all(&runtime).unwrap();
9008 let mut attempt =
9009 RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9010 attempt.jit_received(harness.clock.now()).unwrap();
9011 harness.store.record_attempt(&attempt).unwrap();
9012 harness.clock.advance_secs(11);
9013 let replacements = harness
9014 .launcher
9015 .recover_startup(std::slice::from_ref(&harness.policy))
9016 .await
9017 .unwrap();
9018
9019 let attempts = harness.store.attempts().unwrap();
9020 assert_eq!(attempts.len(), 1);
9021 assert_eq!(
9022 attempts
9023 .iter()
9024 .find(|attempt| attempt.id == id)
9025 .unwrap()
9026 .state(),
9027 AttemptState::Cleaned
9028 );
9029 assert_eq!(
9030 replacements,
9031 vec![ReplacementIntent {
9032 policy: harness.policy.id,
9033 previous_attempt: id,
9034 operation: "jit_expired_replacement",
9035 }]
9036 );
9037 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
9038 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
9039 assert!(harness.delay.0.lock().unwrap().is_empty());
9040 }
9041
9042 #[tokio::test]
9043 async fn package_materialization_retries_are_bounded_and_demand_adjacent() {
9044 let persistent = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9045 persistent.packages.fail_materializations(2);
9046 persistent.ready().await;
9047 persistent.launch().await;
9048 assert_eq!(
9049 persistent.packages.materializations.load(Ordering::SeqCst),
9050 3
9051 );
9052 assert_eq!(
9053 *persistent.delay.0.lock().unwrap(),
9054 vec![Duration::from_millis(10), Duration::from_millis(20)]
9055 );
9056
9057 let gone_before_wait = Harness::new(
9058 FakeGithubLifecycle::default(),
9059 Arc::new(FakeDemand::answering([false])),
9060 );
9061 gone_before_wait.packages.fail_materializations(3);
9062 gone_before_wait.ready().await;
9063 assert!(gone_before_wait.launch_result().await.is_err());
9064 assert_eq!(
9065 gone_before_wait
9066 .packages
9067 .materializations
9068 .load(Ordering::SeqCst),
9069 1
9070 );
9071 assert!(gone_before_wait.delay.0.lock().unwrap().is_empty());
9072
9073 let gone_during_wait = Harness::new(
9074 FakeGithubLifecycle::default(),
9075 Arc::new(FakeDemand::answering([true, false])),
9076 );
9077 gone_during_wait.packages.fail_materializations(3);
9078 gone_during_wait.ready().await;
9079 assert!(gone_during_wait.launch_result().await.is_err());
9080 assert_eq!(
9081 gone_during_wait
9082 .packages
9083 .materializations
9084 .load(Ordering::SeqCst),
9085 1
9086 );
9087 assert_eq!(
9088 *gone_during_wait.delay.0.lock().unwrap(),
9089 vec![Duration::from_millis(10)]
9090 );
9091 }
9092
9093 #[tokio::test]
9094 async fn replacement_is_intent_only_and_never_launches_inside_lifecycle() {
9095 let harness = Harness::new(
9096 FakeGithubLifecycle::default(),
9097 Arc::new(FakeDemand::answering([true, false])),
9098 );
9099 harness.ready().await;
9100 let first = harness.launch().await;
9101 harness.processes.set_alive(false);
9102 harness
9103 .github
9104 .observe(GithubRunnerObservation::NotRegistered);
9105 let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
9106
9107 assert_eq!(harness.store.attempts().unwrap().len(), 1);
9108 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 1);
9109 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 1);
9110 assert!(harness.delay.0.lock().unwrap().is_empty());
9111 assert_eq!(
9112 replacements,
9113 vec![ReplacementIntent {
9114 policy: harness.policy.id,
9115 previous_attempt: first.id,
9116 operation: "exit_before_acceptance_replacement",
9117 }]
9118 );
9119 assert_eq!(
9120 harness.store.attempt(first.id).unwrap().unwrap().state(),
9121 AttemptState::Cleaned
9122 );
9123 }
9124
9125 #[tokio::test]
9126 async fn startup_adopts_a_live_process_and_refuses_launch_before_recovery() {
9127 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9128 let before = harness.launch_result().await;
9129 assert!(before.is_err());
9130 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
9131
9132 let id = AttemptId::new_random();
9133 let runtime = harness.launcher.app_paths.runtime_dir().join("adopt");
9134 fs::create_dir_all(&runtime).unwrap();
9135 let mut attempt =
9136 RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
9137 attempt.jit_received(harness.clock.now()).unwrap();
9138 attempt.started(4242, harness.clock.now()).unwrap();
9139 harness.store.record_attempt(&attempt).unwrap();
9140 harness.processes.set_alive(true);
9141 harness
9142 .github
9143 .observe(GithubRunnerObservation::NotRegistered);
9144 let replacements = harness
9145 .launcher
9146 .recover_startup(std::slice::from_ref(&harness.policy))
9147 .await
9148 .unwrap();
9149 assert!(replacements.is_empty());
9150 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 0);
9151 assert!(
9152 harness
9153 .events
9154 .events()
9155 .contains(&AttemptEvent::Adopted { attempt: id })
9156 );
9157 }
9158
9159 #[tokio::test]
9160 async fn spawn_before_starting_crash_recovers_pid_then_completes_and_cleans() {
9161 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9162 let id = AttemptId::new_random();
9163 let runtime = harness
9164 .launcher
9165 .app_paths
9166 .runtime_dir()
9167 .join("spawn-before-starting");
9168 fs::create_dir_all(&runtime).unwrap();
9169 let mut attempt =
9170 RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9171 attempt.jit_received(harness.clock.now()).unwrap();
9172 harness.store.record_attempt(&attempt).unwrap();
9173 harness.processes.set_alive(true);
9174 harness
9175 .github
9176 .observe(GithubRunnerObservation::Registered { busy: true });
9177
9178 let replacements = harness
9179 .launcher
9180 .recover_startup(std::slice::from_ref(&harness.policy))
9181 .await
9182 .unwrap();
9183 assert!(replacements.is_empty());
9184 let recovered = harness.store.attempt(id).unwrap().unwrap();
9185 assert_eq!(recovered.state(), AttemptState::Busy);
9186 assert_eq!(recovered.process_id(), Some(4242));
9187 assert_eq!(recovered.github_runner_id(), Some(73));
9188 let events = harness.events.events();
9189 let starting = events
9190 .iter()
9191 .position(|event| matches!(event, AttemptEvent::State { attempt, state: AttemptState::Starting } if *attempt == id))
9192 .unwrap();
9193 let busy = events
9194 .iter()
9195 .position(|event| matches!(event, AttemptEvent::State { attempt, state: AttemptState::Busy } if *attempt == id))
9196 .unwrap();
9197 assert!(starting < busy, "recovery skipped a legal edge: {events:?}");
9198
9199 harness.processes.finish_successfully();
9200 harness
9201 .github
9202 .observe(GithubRunnerObservation::NotRegistered);
9203 assert!(
9204 harness
9205 .launcher
9206 .supervise(&harness.policy)
9207 .await
9208 .unwrap()
9209 .is_empty()
9210 );
9211 let cleaned = harness.store.attempt(id).unwrap().unwrap();
9212 assert_eq!(cleaned.state(), AttemptState::Cleaned);
9213 assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::CompletedJob));
9214 assert!(!runtime.exists());
9215 }
9216
9217 #[tokio::test]
9218 async fn failed_post_spawn_stop_keeps_capacity_until_supervision_proves_death() {
9219 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9220 harness.processes.fail_spawn_with_live_child();
9221 harness.ready().await;
9222 assert!(harness.launch_result().await.is_err());
9223
9224 let attempt = harness.only_attempt();
9225 assert_eq!(attempt.state(), AttemptState::Starting);
9226 assert_eq!(attempt.process_id(), Some(4242));
9227 assert!(attempt.outcome().is_none());
9228 assert!(attempt.state().counts_against_capacity());
9229 assert_eq!(harness.processes.spawns.load(Ordering::SeqCst), 1);
9230 assert!(harness.delay.0.lock().unwrap().is_empty());
9231
9232 harness.processes.set_alive(false);
9233 harness
9234 .github
9235 .observe(GithubRunnerObservation::NotRegistered);
9236 let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
9237 assert_eq!(replacements.len(), 1);
9238 assert_eq!(
9239 harness.store.attempt(attempt.id).unwrap().unwrap().state(),
9240 AttemptState::Cleaned
9241 );
9242 }
9243
9244 #[tokio::test]
9245 async fn remote_runner_identity_closes_both_sides_of_the_registration_crash_boundary() {
9246 for sidecar_already_present in [false, true] {
9247 let harness = Harness::new(
9248 FakeGithubLifecycle::default(),
9249 Arc::new(FakeDemand::answering([false])),
9250 );
9251 let id = AttemptId::new_random();
9252 let runtime =
9253 harness
9254 .launcher
9255 .app_paths
9256 .runtime_dir()
9257 .join(if sidecar_already_present {
9258 "after-id-sidecar"
9259 } else {
9260 "before-id-sidecar"
9261 });
9262 fs::create_dir_all(&runtime).unwrap();
9263 let mut attempt =
9264 RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9265 if sidecar_already_present {
9266 write_runner_id(&runtime, 73).unwrap();
9267 attempt.jit_received(harness.clock.now()).unwrap();
9268 }
9269 harness.store.record_attempt(&attempt).unwrap();
9270 harness.processes.set_alive(true);
9271 harness
9272 .github
9273 .observe(GithubRunnerObservation::Registered { busy: false });
9274 harness
9275 .launcher
9276 .recover_startup(std::slice::from_ref(&harness.policy))
9277 .await
9278 .unwrap();
9279
9280 assert_eq!(read_runner_id(&runtime), Some(73));
9281 assert!(
9282 harness
9283 .store
9284 .attempt(id)
9285 .unwrap()
9286 .unwrap()
9287 .outcome()
9288 .is_none()
9289 );
9290 let events = harness.events.events();
9291 let recovered = events.iter().position(|event| {
9292 matches!(
9293 event,
9294 AttemptEvent::RemoteIdentityRecovered {
9295 attempt,
9296 runner_id: 73
9297 } if *attempt == id
9298 )
9299 });
9300 assert_eq!(recovered.is_some(), !sidecar_already_present);
9301 if let Some(recovered) = recovered {
9302 let adopted = events
9303 .iter()
9304 .position(|event| matches!(event, AttemptEvent::Adopted { attempt } if *attempt == id))
9305 .unwrap();
9306 assert!(
9307 recovered < adopted,
9308 "identity was not durable before adoption: {events:?}"
9309 );
9310 }
9311 assert!(runtime.exists());
9312 }
9313 }
9314
9315 #[tokio::test]
9316 async fn recovery_stays_closed_for_unknown_policy_and_unreachable_attempts() {
9317 let unknown = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9318 let unknown_attempt = RunnerAttempt::allocate(
9319 AttemptId::new_random(),
9320 PolicyId::from_u128(0xfeed),
9321 unknown
9322 .launcher
9323 .app_paths
9324 .runtime_dir()
9325 .join("unknown-policy"),
9326 unknown.clock.now(),
9327 );
9328 unknown.store.record_attempt(&unknown_attempt).unwrap();
9329 let expired_id = AttemptId::new_random();
9330 let expired_runtime = unknown
9331 .launcher
9332 .app_paths
9333 .runtime_dir()
9334 .join("expired-beside-unknown");
9335 fs::create_dir_all(&expired_runtime).unwrap();
9336 let mut expired = RunnerAttempt::allocate(
9337 expired_id,
9338 unknown.policy.id,
9339 expired_runtime,
9340 unknown.clock.now(),
9341 );
9342 expired.jit_received(unknown.clock.now()).unwrap();
9343 unknown.store.record_attempt(&expired).unwrap();
9344 unknown.clock.advance_secs(11);
9345 assert!(matches!(
9346 unknown
9347 .launcher
9348 .recover_startup(std::slice::from_ref(&unknown.policy))
9349 .await,
9350 Err(LifecycleError::RecoveryIncomplete)
9351 ));
9352 assert!(unknown.launch_result().await.is_err());
9353 assert_eq!(unknown.processes.spawns.load(Ordering::SeqCst), 0);
9354 let recovered_policy = fixtures::policy()
9355 .id(PolicyId::from_u128(0xfeed))
9356 .repository("octo/repo")
9357 .autoscale("home", 2)
9358 .active()
9359 .build();
9360 let pending = unknown
9361 .launcher
9362 .recover_startup(&[unknown.policy.clone(), recovered_policy])
9363 .await
9364 .unwrap();
9365 assert_eq!(
9366 pending,
9367 vec![ReplacementIntent {
9368 policy: unknown.policy.id,
9369 previous_attempt: expired_id,
9370 operation: "jit_expired_replacement",
9371 }]
9372 );
9373 assert_eq!(unknown.processes.spawns.load(Ordering::SeqCst), 0);
9374
9375 let unreachable = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9376 let id = AttemptId::new_random();
9377 let runtime = unreachable
9378 .launcher
9379 .app_paths
9380 .runtime_dir()
9381 .join("unreachable");
9382 fs::create_dir_all(&runtime).unwrap();
9383 unreachable
9384 .store
9385 .record_attempt(&RunnerAttempt::allocate(
9386 id,
9387 unreachable.policy.id,
9388 runtime,
9389 unreachable.clock.now(),
9390 ))
9391 .unwrap();
9392 unreachable
9393 .github
9394 .observe(GithubRunnerObservation::Unreachable);
9395 assert!(matches!(
9396 unreachable
9397 .launcher
9398 .recover_startup(std::slice::from_ref(&unreachable.policy))
9399 .await,
9400 Err(LifecycleError::RecoveryIncomplete)
9401 ));
9402 assert!(unreachable.launch_result().await.is_err());
9403 assert_eq!(unreachable.processes.spawns.load(Ordering::SeqCst), 0);
9404 }
9405
9406 #[tokio::test]
9407 async fn a_dead_busy_process_unknown_to_github_is_orphaned_and_cleaned() {
9408 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9409 let id = AttemptId::new_random();
9410 let runtime = harness.launcher.app_paths.runtime_dir().join("orphan");
9411 fs::create_dir_all(&runtime).unwrap();
9412 let mut attempt =
9413 RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9414 attempt.jit_received(harness.clock.now()).unwrap();
9415 attempt.started(4242, harness.clock.now()).unwrap();
9416 attempt.assigned_job(73, harness.clock.now()).unwrap();
9417 harness.store.record_attempt(&attempt).unwrap();
9418 harness.processes.set_alive(false);
9419 harness
9420 .github
9421 .observe(GithubRunnerObservation::NotRegistered);
9422 harness
9423 .launcher
9424 .recover_startup(std::slice::from_ref(&harness.policy))
9425 .await
9426 .unwrap();
9427 let cleaned = harness.store.attempt(id).unwrap().unwrap();
9428 assert_eq!(cleaned.state(), AttemptState::Cleaned);
9429 assert_eq!(cleaned.outcome(), Some(&AttemptOutcome::Orphaned));
9430 assert!(!runtime.exists());
9431 }
9432
9433 #[tokio::test]
9434 async fn registration_timeout_journals_intent_stops_then_concludes_with_dead_reason() {
9435 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9436 harness.ready().await;
9437 let id = AttemptId::new_random();
9438 let runtime = harness.launcher.app_paths.runtime_dir().join("timeout");
9439 fs::create_dir_all(&runtime).unwrap();
9440 let mut attempt =
9441 RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
9442 attempt.jit_received(harness.clock.now()).unwrap();
9443 attempt.started(4242, harness.clock.now()).unwrap();
9444 harness.store.record_attempt(&attempt).unwrap();
9445 harness.clock.advance_secs(11);
9446 harness.processes.set_alive(true);
9447 harness
9448 .github
9449 .observe(GithubRunnerObservation::NotRegistered);
9450 let replacements = harness.launcher.supervise(&harness.policy).await.unwrap();
9451 assert_eq!(
9452 replacements,
9453 vec![ReplacementIntent {
9454 policy: harness.policy.id,
9455 previous_attempt: id,
9456 operation: "registration_timeout_replacement",
9457 }]
9458 );
9459
9460 assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 1);
9461 assert!(!harness.processes.alive.load(Ordering::SeqCst));
9462 let actions = harness.processes.actions.lock().unwrap().clone();
9463 let intent = actions
9464 .iter()
9465 .position(|action| *action == "terminate_intent")
9466 .unwrap();
9467 let signal = actions
9468 .iter()
9469 .position(|action| *action == "terminate")
9470 .unwrap();
9471 assert!(
9472 intent < signal,
9473 "intent was not durable before signal: {actions:?}"
9474 );
9475
9476 let cleaned = harness.store.attempt(id).unwrap().unwrap();
9477 assert!(matches!(
9478 cleaned.outcome(),
9479 Some(AttemptOutcome::Failed {
9480 reason: FailureReason::TerminatedAfterRegistrationTimeout
9481 })
9482 ));
9483 let events = harness.events.events();
9484 let intent = events
9485 .iter()
9486 .position(|event| matches!(event, AttemptEvent::TerminateIntent { .. }))
9487 .unwrap();
9488 let stopped = events
9489 .iter()
9490 .position(|event| matches!(event, AttemptEvent::Terminated { .. }))
9491 .unwrap();
9492 let concluded = events
9493 .iter()
9494 .position(|event| matches!(event, AttemptEvent::Concluded { .. }))
9495 .unwrap();
9496 assert!(intent < stopped && stopped < concluded, "{events:?}");
9497 }
9498
9499 #[tokio::test]
9500 async fn timeout_crash_recovery_returns_the_same_replacement_intent() {
9501 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9502 let id = AttemptId::new_random();
9503 let runtime = harness
9504 .launcher
9505 .app_paths
9506 .runtime_dir()
9507 .join("timeout-after-crash");
9508 fs::create_dir_all(&runtime).unwrap();
9509 let mut attempt =
9510 RunnerAttempt::allocate(id, harness.policy.id, runtime, harness.clock.now());
9511 attempt.jit_received(harness.clock.now()).unwrap();
9512 attempt.started(4242, harness.clock.now()).unwrap();
9513 harness.store.record_attempt(&attempt).unwrap();
9514 harness.processes.intent.store(true, Ordering::SeqCst);
9515 harness.processes.set_alive(false);
9516 harness
9517 .github
9518 .observe(GithubRunnerObservation::NotRegistered);
9519
9520 let replacements = harness
9521 .launcher
9522 .recover_startup(std::slice::from_ref(&harness.policy))
9523 .await
9524 .unwrap();
9525 assert_eq!(
9526 replacements,
9527 vec![ReplacementIntent {
9528 policy: harness.policy.id,
9529 previous_attempt: id,
9530 operation: "registration_timeout_replacement",
9531 }]
9532 );
9533 let consumed = RunnerLauncher::supervise(&harness.launcher, &harness.policy)
9534 .await
9535 .unwrap();
9536 assert_eq!(consumed, replacements);
9537 assert!(
9538 RunnerLauncher::supervise(&harness.launcher, &harness.policy)
9539 .await
9540 .unwrap()
9541 .is_empty(),
9542 "startup replacement evidence must be consumed exactly once by e1"
9543 );
9544 assert!(matches!(
9545 harness.store.attempt(id).unwrap().unwrap().outcome(),
9546 Some(AttemptOutcome::Failed {
9547 reason: FailureReason::TerminatedAfterRegistrationTimeout
9548 })
9549 ));
9550 }
9551
9552 #[tokio::test]
9553 async fn terminate_intent_sync_failure_prevents_signal_and_conclusion() {
9554 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9555 let id = AttemptId::new_random();
9556 let runtime = harness
9557 .launcher
9558 .app_paths
9559 .runtime_dir()
9560 .join("timeout-sync-failure");
9561 fs::create_dir_all(&runtime).unwrap();
9562 let mut attempt =
9563 RunnerAttempt::allocate(id, harness.policy.id, &runtime, harness.clock.now());
9564 attempt.jit_received(harness.clock.now()).unwrap();
9565 attempt.started(4242, harness.clock.now()).unwrap();
9566 harness.store.record_attempt(&attempt).unwrap();
9567 harness.clock.advance_secs(11);
9568 harness.processes.set_alive(true);
9569 harness.processes.fail_intent();
9570 harness
9571 .github
9572 .observe(GithubRunnerObservation::NotRegistered);
9573
9574 assert!(
9575 harness
9576 .launcher
9577 .recover_startup(std::slice::from_ref(&harness.policy))
9578 .await
9579 .is_err()
9580 );
9581 assert_eq!(harness.processes.terminations.load(Ordering::SeqCst), 0);
9582 assert!(harness.processes.alive.load(Ordering::SeqCst));
9583 assert_eq!(
9584 harness.store.attempt(id).unwrap().unwrap().state(),
9585 AttemptState::Starting
9586 );
9587 assert!(!harness.events.events().iter().any(|event| matches!(
9588 event,
9589 AttemptEvent::Terminated { attempt } | AttemptEvent::Concluded { attempt, .. }
9590 if *attempt == id
9591 )));
9592 }
9593
9594 #[tokio::test]
9595 async fn diagnostics_survive_cleanup_without_the_jit_or_a_token() {
9596 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9597 harness.ready().await;
9598 let attempt = harness.launch().await;
9599 harness
9600 .github
9601 .observe(GithubRunnerObservation::Registered { busy: false });
9602 harness.launcher.supervise(&harness.policy).await.unwrap();
9603 harness.clock.advance_secs(11);
9604 harness.processes.set_alive(false);
9605 harness
9606 .github
9607 .observe(GithubRunnerObservation::NotRegistered);
9608 harness.launcher.supervise(&harness.policy).await.unwrap();
9609 let diagnostic = fs::read_to_string(
9610 harness
9611 .launcher
9612 .diagnostics_root
9613 .join(format!("{}.log", attempt.id)),
9614 )
9615 .unwrap();
9616 assert!(diagnostic.contains("exited_idle_without_work"));
9617 assert!(!diagnostic.contains(JIT));
9618 assert!(!diagnostic.contains("ghp_"));
9619 assert!(!attempt.runtime_path().exists());
9620 }
9621
9622 #[test]
9623 fn native_process_listing_never_contains_jit_and_handoffs_never_survive() {
9624 let root = tempfile::tempdir().unwrap();
9625 let policy = fixtures::policy()
9626 .repository("octo/repo")
9627 .autoscale("home", 1)
9628 .active()
9629 .build();
9630 let runtime = root.path().join("successful");
9631 fs::create_dir_all(&runtime).unwrap();
9632 let processes = NativeProcesses::new();
9633 let config = EncodedJitConfig::new(JIT);
9634 let handoff =
9635 RestrictiveHandoff::create(&runtime, SecretString::from(config.expose().to_owned()))
9636 .unwrap();
9637 let mut child = native_inspection_spec()
9638 .spawn_runner_with_handoff(&handoff)
9639 .expect("native child starts");
9640 let pid = child.pid();
9641 handoff.delete().unwrap();
9642 let command_line = native_command_line(pid);
9643 assert!(
9644 !command_line.contains(JIT),
9645 "the encoded JIT configuration appeared in the native process listing"
9646 );
9647 assert_no_jit_file(&runtime);
9648 child
9649 .stop(Duration::from_secs(1))
9650 .expect("native child stops");
9651
9652 let failed_runtime = root.path().join("failed");
9653 fs::create_dir_all(&failed_runtime).unwrap();
9654 let failed = RunnerAttempt::allocate(
9655 AttemptId::new_random(),
9656 policy.id,
9657 &failed_runtime,
9658 FakeClock::default().now(),
9659 );
9660 assert!(
9661 processes
9662 .spawn(&failed, &EncodedJitConfig::new(JIT))
9663 .is_err(),
9664 "a runtime with no runner executable must fail"
9665 );
9666 assert_no_jit_file(&failed_runtime);
9667 processes
9668 .record_terminate_intent(&failed)
9669 .expect("the intent file and its directory entry are durably synced");
9670 assert_eq!(
9671 fs::read(NativeProcesses::intent_path(&failed)).unwrap(),
9672 b"registration-timeout\n"
9673 );
9674 }
9675
9676 #[test]
9677 fn post_spawn_boundaries_are_bounded_durable_and_never_retry_jit() {
9678 let root = tempfile::tempdir().unwrap();
9679 let policy = fixtures::policy()
9680 .repository("octo/repo")
9681 .autoscale("home", 1)
9682 .active()
9683 .build();
9684 let processes = NativeProcesses::new();
9685 processes.use_long_lived_test_listener();
9686 for (index, boundary) in [
9687 PostSpawnBoundary::HandoffDelete,
9688 PostSpawnBoundary::IdentitySerialize,
9689 PostSpawnBoundary::IdentityWrite,
9690 PostSpawnBoundary::ChildMapInsert,
9691 ]
9692 .into_iter()
9693 .enumerate()
9694 {
9695 let runtime = root.path().join(format!("post-spawn-{index}"));
9696 let bin = runtime.join("bin");
9697 fs::create_dir_all(&bin).unwrap();
9698 #[cfg(windows)]
9699 let listener = bin.join("Runner.Listener.exe");
9700 #[cfg(not(windows))]
9701 let listener = bin.join("Runner.Listener");
9702 fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9703 let attempt = RunnerAttempt::allocate(
9704 AttemptId::new_random(),
9705 policy.id,
9706 &runtime,
9707 FakeClock::default().now(),
9708 );
9709 processes.fail_post_spawn_at(boundary);
9710 let failure = processes
9711 .spawn(&attempt, &EncodedJitConfig::new(JIT))
9712 .expect_err("fault must cross the post-spawn cleanup path");
9713 assert!(!failure.retryable, "{boundary:?} allowed duplicate retry");
9714 assert!(
9715 !processes.is_alive(&attempt).unwrap(),
9716 "{boundary:?} left a child"
9717 );
9718 assert!(!NativeProcesses::identity_path(&attempt).exists());
9719 assert_no_jit_file(&runtime);
9720 }
9721 assert_eq!(processes.post_spawn_reaps.load(Ordering::SeqCst), 4);
9722
9723 let runtime = root.path().join("identity-and-stop-fail");
9724 let bin = runtime.join("bin");
9725 fs::create_dir_all(&bin).unwrap();
9726 #[cfg(windows)]
9727 let listener = bin.join("Runner.Listener.exe");
9728 #[cfg(not(windows))]
9729 let listener = bin.join("Runner.Listener");
9730 fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9731 let attempt = RunnerAttempt::allocate(
9732 AttemptId::new_random(),
9733 policy.id,
9734 &runtime,
9735 FakeClock::default().now(),
9736 );
9737 processes.fail_post_spawn_at(PostSpawnBoundary::IdentityWrite);
9741 processes.fail_post_spawn_at(PostSpawnBoundary::IdentityWrite);
9742 processes.fail_next_post_spawn_stop();
9743 let failure = processes
9744 .spawn(&attempt, &EncodedJitConfig::new(JIT))
9745 .expect_err("the identity boundary must fail closed");
9746 assert!(failure.live_pid.is_some());
9747 assert_long_lived_listener_ready(&processes, &attempt);
9748 assert!(processes.is_alive(&attempt).unwrap());
9749 assert!(!NativeProcesses::identity_path(&attempt).exists());
9750 assert!(NativeProcesses::fallback_identity_path(&attempt).is_file());
9751 assert_eq!(processes.post_spawn_reaps.load(Ordering::SeqCst), 4);
9752 processes.terminate(&attempt).unwrap();
9753
9754 let runtime = root.path().join("persistent-stop-and-identity-failures");
9755 let bin = runtime.join("bin");
9756 fs::create_dir_all(&bin).unwrap();
9757 #[cfg(windows)]
9758 let listener = bin.join("Runner.Listener.exe");
9759 #[cfg(not(windows))]
9760 let listener = bin.join("Runner.Listener");
9761 fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9762 let mut unresolved = RunnerAttempt::allocate(
9763 AttemptId::new_random(),
9764 policy.id,
9765 &runtime,
9766 FakeClock::default().now(),
9767 );
9768 for _ in 0..3 {
9769 processes.fail_post_spawn_at(PostSpawnBoundary::IdentityWrite);
9770 }
9771 processes.fail_post_spawn_stops(MAX_POST_SPAWN_STOP_ATTEMPTS);
9772 let failure = processes
9773 .spawn(&unresolved, &EncodedJitConfig::new(JIT))
9774 .expect_err("bounded cleanup must return even when every stop errors");
9775 let pid = failure
9776 .live_pid
9777 .expect("the owned child remains supervised in this invocation");
9778 assert!(matches!(failure.reason, FailureReason::Other(_)));
9779 assert_long_lived_listener_ready(&processes, &unresolved);
9780 unresolved.jit_received(FakeClock::default().now()).unwrap();
9781 unresolved.started(pid, FakeClock::default().now()).unwrap();
9782 let journal = SqliteStore::open_in_memory().unwrap();
9783 journal.record_attempt(&unresolved).unwrap();
9784 let recovered = journal.attempt(unresolved.id).unwrap().unwrap();
9785 assert_eq!(recovered.process_id(), Some(pid));
9786 assert_eq!(recovered.state(), AttemptState::Starting);
9787 assert!(processes.is_alive(&unresolved).unwrap());
9788 assert!(!NativeProcesses::identity_path(&unresolved).exists());
9789 assert!(!NativeProcesses::fallback_identity_path(&unresolved).exists());
9790 assert_eq!(
9791 fs::read_to_string(NativeProcesses::unresolved_process_path(&unresolved)).unwrap(),
9792 pid.to_string(),
9793 "bounded cleanup must leave durable unresolved-process evidence before returning"
9794 );
9795 assert!(
9796 NativeProcesses::new().is_alive(&recovered).is_err(),
9797 "restart must fail closed on the durable starting/PID journal rather than trust a bare PID"
9798 );
9799 processes.terminate(&unresolved).unwrap();
9800
9801 let runtime = root.path().join("post-spawn-stop-failed");
9802 let bin = runtime.join("bin");
9803 fs::create_dir_all(&bin).unwrap();
9804 #[cfg(windows)]
9805 let listener = bin.join("Runner.Listener.exe");
9806 #[cfg(not(windows))]
9807 let listener = bin.join("Runner.Listener");
9808 fs::copy(std::env::current_exe().unwrap(), &listener).unwrap();
9809 let attempt = RunnerAttempt::allocate(
9810 AttemptId::new_random(),
9811 policy.id,
9812 &runtime,
9813 FakeClock::default().now(),
9814 );
9815 processes.fail_post_spawn_at(PostSpawnBoundary::ChildMapInsert);
9816 processes.fail_next_post_spawn_stop();
9817 let failure = processes
9818 .spawn(&attempt, &EncodedJitConfig::new(JIT))
9819 .expect_err("the injected stop failure must preserve supervision");
9820 let live_pid = failure
9821 .live_pid
9822 .expect("live PID is returned to the journal");
9823 assert!(!failure.retryable);
9824 assert_long_lived_listener_ready(&processes, &attempt);
9825 assert!(NativeProcesses::identity_path(&attempt).is_file());
9826 assert_eq!(
9827 NativeProcesses::new()
9828 .read_identity(&attempt)
9829 .unwrap()
9830 .runner()
9831 .unwrap()
9832 .pid(),
9833 live_pid
9834 );
9835 assert_eq!(processes.post_spawn_reaps.load(Ordering::SeqCst), 4);
9836 processes.terminate(&attempt).unwrap();
9837 }
9838
9839 #[test]
9840 #[ignore = "spawned only as the platform-stable native listener fixture"]
9841 fn long_lived_native_listener_helper() {
9842 let ready = std::env::var_os("RUNNER_MANAGER_TEST_LISTENER_READY")
9843 .map(PathBuf::from)
9844 .expect("the parent supplies the readiness path");
9845 runner_manager_platform::host_fitness::write_atomically(&ready, b"ready\n")
9848 .expect("the listener publishes readiness");
9849 std::thread::sleep(Duration::from_secs(30));
9850 }
9851
9852 fn assert_long_lived_listener_ready(processes: &NativeProcesses, attempt: &RunnerAttempt) {
9853 let ready = attempt.runtime_path().join(TEST_LISTENER_READY);
9854 let deadline = std::time::Instant::now() + Duration::from_secs(5);
9855 loop {
9856 if ready.is_file() {
9857 assert_eq!(fs::read(&ready).unwrap(), b"ready\n");
9858 return;
9859 }
9860 assert!(
9861 processes.is_alive(attempt).unwrap(),
9862 "the native listener exited before publishing readiness"
9863 );
9864 assert!(
9865 std::time::Instant::now() < deadline,
9866 "the native listener stayed alive but never published readiness"
9867 );
9868 std::thread::sleep(Duration::from_millis(10));
9869 }
9870 }
9871
9872 #[tokio::test]
9873 async fn a_reservation_is_journalled_before_any_effect_and_holds_its_slot() {
9874 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9875 harness.ready().await;
9876
9877 let reserved = harness.reserve().await.unwrap();
9878
9879 assert_eq!(reserved.state(), AttemptState::Allocated);
9880 assert!(reserved.counts_against_capacity());
9881 let journalled = harness.store.attempt(reserved.id).unwrap().unwrap();
9882 assert_eq!(journalled.state(), AttemptState::Allocated);
9883 assert_eq!(harness.packages.materializations.load(Ordering::SeqCst), 0);
9884 assert_eq!(harness.github.registrations.load(Ordering::SeqCst), 0);
9885
9886 let started = harness
9887 .launcher
9888 .finish_launch(&harness.policy, reserved, ReservationWaits::default())
9889 .await
9890 .unwrap();
9891 assert_eq!(started.state(), AttemptState::Starting);
9892 }
9893
9894 #[tokio::test]
9898 async fn a_pass_leaves_a_runner_coming_up_alone_and_supervises_it_once_up() {
9899 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9900 harness.ready().await;
9901 let reserved = harness.reserve().await.unwrap();
9902 let looks_left = || harness.github.observations.lock().unwrap().len();
9903 harness
9904 .github
9905 .observations
9906 .lock()
9907 .unwrap()
9908 .push_back(LifecycleGithubObservation::not_registered());
9909
9910 harness.launcher.supervise(&harness.policy).await.unwrap();
9911 assert_eq!(looks_left(), 1, "it did not look");
9912 let journalled = harness.store.attempt(reserved.id).unwrap().unwrap();
9913 assert_eq!(journalled.state(), AttemptState::Allocated);
9914
9915 harness
9916 .launcher
9917 .finish_launch(&harness.policy, reserved, ReservationWaits::default())
9918 .await
9919 .unwrap();
9920 assert!(harness.launcher.bringing_up.lock().unwrap().is_empty());
9921
9922 harness.launcher.supervise(&harness.policy).await.unwrap();
9923 assert_eq!(looks_left(), 0, "up, it is this pass's to supervise");
9924 }
9925
9926 #[tokio::test]
9929 async fn a_crash_between_reservation_and_spawn_recovers_the_slot() {
9930 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9931 harness.ready().await;
9932 let reserved = harness.reserve().await.unwrap();
9933 harness.clock.advance_secs(31);
9934
9935 harness
9936 .launcher
9937 .recover_startup(std::slice::from_ref(&harness.policy))
9938 .await
9939 .unwrap();
9940
9941 let recovered = harness.store.attempt(reserved.id).unwrap().unwrap();
9942 assert!(
9943 !recovered.counts_against_capacity(),
9944 "the slot is free again: {:?}",
9945 recovered.state()
9946 );
9947 }
9948
9949 #[tokio::test]
9951 async fn a_crash_after_the_package_is_copied_recovers_the_slot() {
9952 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9953 harness.ready().await;
9954 let reserved = harness.reserve().await.unwrap();
9955 harness
9956 .packages
9957 .materialize(&reserved, &mut PackageTimings::default())
9958 .await
9959 .unwrap();
9960 harness.clock.advance_secs(31);
9961
9962 harness
9963 .launcher
9964 .recover_startup(std::slice::from_ref(&harness.policy))
9965 .await
9966 .unwrap();
9967
9968 let recovered = harness.store.attempt(reserved.id).unwrap().unwrap();
9969 assert!(!recovered.counts_against_capacity());
9970 }
9971
9972 #[tokio::test]
9973 async fn runners_prepare_at_once_within_the_bound() {
9974 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
9975 harness.ready().await;
9976 let turns = harness.launcher.turns.available_permits();
9977 harness.launcher.turns.forget_permits(turns - 2);
9978 let mut reserved = Vec::new();
9979 for _ in 0..4 {
9980 reserved.push(harness.reserve().await.unwrap());
9981 }
9982
9983 let finished = futures::future::join_all(reserved.into_iter().map(|attempt| {
9984 harness
9985 .launcher
9986 .finish_launch(&harness.policy, attempt, ReservationWaits::default())
9987 }))
9988 .await;
9989
9990 assert!(finished.iter().all(Result::is_ok));
9991 assert_eq!(
9992 harness.packages.peak_materializing.load(Ordering::SeqCst),
9993 2
9994 );
9995 }
9996
9997 #[tokio::test]
9998 async fn every_production_launch_prunes_under_the_same_allocation_guard() {
9999 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
10000 harness.ready().await;
10001 assert_eq!(harness.packages.prunes.load(Ordering::SeqCst), 0);
10002 harness.launch().await;
10003 assert_eq!(harness.packages.prunes.load(Ordering::SeqCst), 1);
10004 }
10005
10006 fn copy_package_tree(
10007 source: &Path,
10008 destination: &Path,
10009 seeds: &TrustedTrees,
10010 ) -> std::io::Result<()> {
10011 let origin = copy_origin(source, destination, seeds);
10012 copy_package_from(source, &origin, destination, seeds)
10013 }
10014
10015 fn assert_no_jit_file(runtime: &Path) {
10016 for entry in fs::read_dir(runtime).unwrap() {
10017 let path = entry.unwrap().path();
10018 if path.is_file() {
10019 let bytes = fs::read(&path).unwrap();
10020 assert!(
10021 !bytes
10022 .windows(JIT.len())
10023 .any(|window| window == JIT.as_bytes()),
10024 "a JIT payload survived in a runtime file"
10025 );
10026 }
10027 }
10028 }
10029
10030 #[test]
10031 fn production_listener_command_uses_the_supported_jit_contract() {
10032 let runtime = Path::new("runtime");
10033 let spec = runner_listener_spec(
10034 PathBuf::from("Runner.Listener"),
10035 runtime,
10036 Vec::new(),
10037 &RunnerEnv::default(),
10038 );
10039 let arguments: Vec<_> = spec
10040 .arguments()
10041 .iter()
10042 .map(|argument| argument.to_string_lossy().into_owned())
10043 .collect();
10044
10045 assert_eq!(arguments, ["run"]);
10046 assert!(
10047 !arguments
10048 .iter()
10049 .any(|argument| argument == "--jit-config-file"),
10050 "the obsolete file option would be rejected by Runner.Listener 2.336.0"
10051 );
10052 }
10053
10054 #[test]
10058 fn production_listener_layers_runner_env_over_the_platform_defaults() {
10059 let root = tempfile::tempdir().unwrap();
10060 let runtime = root.path().join("attempt");
10061 let host_env =
10062 RunnerEnv::parse("ELECTRON_CACHE=/cache/electron\nHOME=/operator/chose/this\n")
10063 .unwrap();
10064 let caches = vec![
10065 (
10066 "ELECTRON_CACHE",
10067 std::ffi::OsString::from("/rman/_cache/o/r/electron"),
10068 ),
10069 (
10070 "npm_config_cache",
10071 std::ffi::OsString::from("/rman/_cache/o/r/npm"),
10072 ),
10073 ];
10074 let spec = runner_listener_spec(
10075 PathBuf::from("Runner.Listener"),
10076 &runtime,
10077 caches,
10078 &host_env,
10079 );
10080 let env = |name: &str| spec.configured_env(name).map(PathBuf::from);
10081
10082 assert!(spec.expects_normal_scheduling());
10083 assert_eq!(
10084 env("ELECTRON_CACHE"),
10085 Some(PathBuf::from("/cache/electron")),
10086 "runner.env must win over a dependency cache of the same name"
10087 );
10088 assert_eq!(
10089 env("npm_config_cache"),
10090 Some(PathBuf::from("/rman/_cache/o/r/npm"))
10091 );
10092 assert_eq!(
10093 env("HOME"),
10094 Some(PathBuf::from("/operator/chose/this")),
10095 "runner.env must win over a platform default of the same name"
10096 );
10097 for tmp in ["TMPDIR", "TEMP", "TMP"] {
10098 assert_eq!(env(tmp), Some(runtime.join("tmp")), "{tmp}");
10099 }
10100
10101 let home = runtime.join(runner_env::RUNNER_HOME_DIR);
10102 if cfg!(windows) {
10103 assert_eq!(env("USERPROFILE"), Some(home.clone()));
10106 assert_eq!(env("APPDATA"), Some(home.join(r"AppData\Roaming")));
10107 assert_eq!(env("LOCALAPPDATA"), Some(home.join(r"AppData\Local")));
10108 assert!(home.join(r"AppData\Roaming").is_dir());
10109 assert!(home.join(r"AppData\Local").is_dir());
10110 } else {
10111 assert_eq!(env("USERPROFILE"), None);
10113 assert!(!home.exists());
10114 }
10115 }
10116
10117 #[test]
10121 fn production_listener_opts_powershell_out_of_telemetry() {
10122 let root = tempfile::tempdir().unwrap();
10123 let runtime = root.path().join("attempt");
10124 let listener = |host_env: &RunnerEnv| {
10125 runner_listener_spec(
10126 PathBuf::from("Runner.Listener"),
10127 &runtime,
10128 Vec::new(),
10129 host_env,
10130 )
10131 };
10132 let optout = |spec: &SpawnSpec| {
10133 spec.configured_env(runner_env::POWERSHELL_TELEMETRY_OPTOUT)
10134 .map(OsStr::to_os_string)
10135 };
10136
10137 let spec = listener(&RunnerEnv::default());
10138 if std::env::var_os(runner_env::POWERSHELL_TELEMETRY_OPTOUT)
10139 .is_some_and(|value| !value.is_empty())
10140 {
10141 assert_eq!(optout(&spec), None);
10143 } else {
10144 assert_eq!(
10145 optout(&spec),
10146 Some(runner_env::POWERSHELL_TELEMETRY_OPTOUT_VALUE.into())
10147 );
10148 }
10149
10150 let operator = RunnerEnv::parse("POWERSHELL_TELEMETRY_OPTOUT=0\n").unwrap();
10151 assert_eq!(
10152 optout(&listener(&operator)),
10153 Some("0".into()),
10154 "runner.env must win over the telemetry opt-out"
10155 );
10156 }
10157
10158 #[test]
10159 fn an_invalid_runner_env_fails_the_launch_without_echoing_a_value() {
10160 let root = tempfile::tempdir().unwrap();
10161 let path = runner_env::path_in(root.path());
10162 fs::write(&path, "GOOD=1\nghp_notarealtokenbutshapedlikeone\n").unwrap();
10163 let processes = NativeProcesses::new().with_runner_env_file(path.clone());
10164 let failure = processes.host_env().unwrap_err().to_string();
10165 assert!(
10166 failure.contains("runner.env") && failure.contains("line 2"),
10167 "{failure}"
10168 );
10169 assert!(
10170 !failure.contains("ghp_"),
10171 "the failure echoed a value: {failure}"
10172 );
10173
10174 let policy = fixtures::policy()
10177 .repository("octo/repo")
10178 .autoscale("home", 1)
10179 .active()
10180 .build();
10181 let attempt = RunnerAttempt::allocate(
10182 AttemptId::new_random(),
10183 policy.id,
10184 root.path(),
10185 FakeClock::default().now(),
10186 );
10187 assert!(processes.prepare(&attempt, &policy).is_err());
10188
10189 fs::write(&path, "GOOD=1\n").unwrap();
10190 assert!(processes.prepare(&attempt, &policy).is_ok());
10191 assert_eq!(processes.host_env().unwrap().get("GOOD"), Some("1"));
10192 assert!(NativeProcesses::new().host_env().unwrap().is_empty());
10193 }
10194
10195 #[test]
10199 fn prepare_hands_the_policys_cache_namespace_to_the_launch() {
10200 let root = tempfile::tempdir().unwrap();
10201 let runner_root = root.path().join("rman");
10202 let runtime = runner_root.join("a1");
10203 fs::create_dir_all(runtime.join("bin")).unwrap();
10204 let config = runner_manager_platform::dependency_cache::config_path_in(root.path());
10205 let source_root = runner_root.clone();
10206 let processes = NativeProcesses::new().with_dependency_caches(DependencyCaches::new(
10207 config.clone(),
10208 Arc::new(move || Some(source_root.clone())),
10209 None,
10210 ));
10211 let policy = fixtures::policy()
10212 .repository("Octo/Repo")
10213 .autoscale("home", 1)
10214 .active()
10215 .build();
10216 let attempt = RunnerAttempt::allocate(
10217 AttemptId::new_random(),
10218 policy.id,
10219 &runtime,
10220 FakeClock::default().now(),
10221 );
10222 assert!(
10223 processes
10224 .cache_env(&attempt, None, &RunnerEnv::default())
10225 .is_empty(),
10226 "a launch with no prepared target, such as a bare `spawn`, has no cache"
10227 );
10228 let prepared = processes.prepare(&attempt, &policy).unwrap();
10229 let variables = processes.cache_env(
10230 &attempt,
10231 prepared.cache_target.as_ref(),
10232 &RunnerEnv::default(),
10233 );
10234 let npm = variables
10235 .iter()
10236 .find(|(name, _)| *name == "npm_config_cache")
10237 .map(|(_, value)| PathBuf::from(value));
10238 assert_eq!(
10239 npm,
10240 Some(
10241 runner_root
10242 .join("_cache")
10243 .join("octo")
10244 .join("repo")
10245 .join("npm")
10246 )
10247 );
10248
10249 fs::write(&config, "[tools]\nnot-a-tool = true\n").unwrap();
10250 let refused = processes
10251 .prepare(&attempt, &policy)
10252 .unwrap_err()
10253 .to_string();
10254 assert!(refused.contains("caches.toml"), "{refused}");
10255 }
10256
10257 #[cfg(windows)]
10258 fn native_inspection_spec() -> SpawnSpec {
10259 SpawnSpec::new("powershell.exe").args([
10260 "-NoProfile",
10261 "-NonInteractive",
10262 "-Command",
10263 "Start-Sleep -Seconds 30",
10264 ])
10265 }
10266
10267 #[cfg(unix)]
10268 fn native_inspection_spec() -> SpawnSpec {
10269 SpawnSpec::new("/bin/sh").args(["-c", "sleep 30"])
10270 }
10271
10272 #[cfg(windows)]
10273 fn native_command_line(pid: u32) -> String {
10274 let output = std::process::Command::new("powershell.exe")
10275 .args([
10276 "-NoProfile",
10277 "-NonInteractive",
10278 "-Command",
10279 &format!("(Get-CimInstance Win32_Process -Filter 'ProcessId = {pid}').CommandLine"),
10280 ])
10281 .output()
10282 .expect("PowerShell can inspect the native child");
10283 assert!(output.status.success(), "native process inspection failed");
10284 String::from_utf8(output.stdout).expect("Windows command lines are Unicode")
10285 }
10286
10287 #[cfg(target_os = "linux")]
10288 fn native_command_line(pid: u32) -> String {
10289 fs::read(format!("/proc/{pid}/cmdline"))
10290 .map(|bytes| String::from_utf8_lossy(&bytes).replace('\0', " "))
10291 .expect("/proc exposes the native child command line")
10292 }
10293
10294 #[cfg(target_os = "macos")]
10295 fn native_command_line(pid: u32) -> String {
10296 let output = std::process::Command::new("ps")
10297 .args(["-o", "command=", "-p", &pid.to_string()])
10298 .output()
10299 .expect("ps can inspect the native child");
10300 assert!(output.status.success(), "native process inspection failed");
10301 String::from_utf8(output.stdout).expect("the command line is UTF-8")
10302 }
10303
10304 #[tokio::test]
10307 async fn a_persistent_repository_leases_s1_and_journals_it_before_any_github_effect() {
10308 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10309 .with_persistent_workspace(2);
10310 harness.github.watch_journal(Arc::clone(&harness.store));
10311 harness.ready().await;
10312
10313 let attempt = harness.launch().await;
10314
10315 assert_eq!(
10316 attempt.workspace(),
10317 AttemptWorkspace::persistent_slot(nz(1)),
10318 "the lowest free slot is leased"
10319 );
10320 assert_eq!(attempt.runtime_path(), harness.slot_path(1));
10321 assert!(attempt.holds_slot_lease());
10322 assert_eq!(
10324 harness.attempt(attempt.id).runtime_path(),
10325 harness.slot_path(1)
10326 );
10327
10328 let facts = harness.github.registration_facts();
10332 assert_eq!(facts.len(), 1);
10333 assert_eq!(
10334 facts[0].leased_slots,
10335 vec![1],
10336 "the lease was journalled first"
10337 );
10338 assert_eq!(facts[0].work_folder, DEFAULT_WORK_FOLDER);
10339 }
10340
10341 #[tokio::test]
10342 async fn a_terminal_but_uncleaned_attempt_keeps_its_slot_without_holding_capacity() {
10343 let harness = Harness::new(
10344 FakeGithubLifecycle::default().fail(true),
10345 Arc::new(PersistentDemand),
10346 )
10347 .with_persistent_workspace(2);
10348 harness.ready().await;
10349
10350 harness.launch_result().await.unwrap_err();
10352 let first = harness.store.attempts().unwrap().remove(0);
10353 assert_eq!(first.state(), AttemptState::Failed);
10354 assert!(
10355 !first.state().counts_against_capacity(),
10356 "a concluded attempt is invisible to host capacity"
10357 );
10358 assert!(
10359 first.holds_slot_lease(),
10360 "and still owns its directory, so its slot is not free"
10361 );
10362
10363 let second = harness.launch().await;
10364 assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
10365 assert_eq!(second.runtime_path(), harness.slot_path(2));
10366 assert_eq!(
10367 harness
10368 .store
10369 .slot_leases_for_policy(harness.policy.id)
10370 .unwrap()
10371 .len(),
10372 2
10373 );
10374 }
10375
10376 #[tokio::test]
10377 async fn two_sequential_allocations_at_capacity_one_reuse_s1_and_its_retained_work() {
10378 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10379 .with_persistent_workspace(1);
10380 harness.ready().await;
10381
10382 let first = harness.launch().await;
10383 assert_eq!(first.runtime_path(), harness.slot_path(1));
10384
10385 let checkout = harness.slot_path(1).join(DEFAULT_WORK_FOLDER).join("repo");
10387 fs::create_dir_all(&checkout).unwrap();
10388 fs::write(checkout.join("checkout.txt"), b"from the first job").unwrap();
10389
10390 harness.cleanup_retaining_work(first.id).await;
10391
10392 let second = harness.launch().await;
10393 assert_ne!(second.id, first.id);
10394 assert_eq!(
10395 second.workspace(),
10396 AttemptWorkspace::persistent_slot(nz(1)),
10397 "a released slot is leased again rather than skipped"
10398 );
10399 assert_eq!(
10400 second.runtime_path(),
10401 first.runtime_path(),
10402 "the same slot is the same exact path"
10403 );
10404 assert_eq!(
10405 fs::read_to_string(checkout.join("checkout.txt")).unwrap(),
10406 "from the first job",
10407 "the retained job workspace survived the second allocation"
10408 );
10409 assert!(harness.slot_path(1).join("runner-package").exists());
10411 }
10412
10413 #[tokio::test]
10414 async fn lowering_capacity_leaves_higher_slots_alone_and_raising_it_permits_them_again() {
10415 let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10416 .with_persistent_workspace(2);
10417 harness.ready().await;
10418
10419 let first = harness.launch().await;
10420 let second = harness.launch().await;
10421 assert_eq!(second.runtime_path(), harness.slot_path(2));
10422 let kept = harness
10423 .slot_path(2)
10424 .join(DEFAULT_WORK_FOLDER)
10425 .join("kept.txt");
10426 fs::create_dir_all(kept.parent().unwrap()).unwrap();
10427 fs::write(&kept, b"s2 was here").unwrap();
10428 harness.cleanup_retaining_work(second.id).await;
10429
10430 harness.policy.set_max_capacity(nz(1)).unwrap();
10432 let refusal = harness.launch_result().await.unwrap_err().to_string();
10433 assert!(
10434 refusal.contains("s1 to s1"),
10435 "the refusal names the ceiling it reached: {refusal}"
10436 );
10437 assert!(
10438 harness.slot_path(2).exists() && kept.exists(),
10439 "lowering capacity deletes nothing; the higher slot is merely unusable"
10440 );
10441
10442 harness.policy.set_max_capacity(nz(2)).unwrap();
10444 let third = harness.launch().await;
10445 assert_eq!(third.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
10446 assert_eq!(third.runtime_path(), harness.slot_path(2));
10447 assert_eq!(fs::read_to_string(&kept).unwrap(), "s2 was here");
10448 assert!(first.holds_slot_lease(), "s1 was never disturbed");
10449 }
10450
10451 #[tokio::test]
10452 async fn organization_and_ephemeral_policies_never_enter_slot_allocation() {
10453 for policy in [
10454 fixtures::policy()
10455 .organization("octo")
10456 .autoscale("home", 2)
10457 .active()
10458 .build(),
10459 fixtures::policy()
10460 .repository("octo/repo")
10461 .autoscale("home", 2)
10462 .active()
10463 .build(),
10464 ] {
10465 let mut harness =
10466 Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10467 .with_host_runner_root();
10468 assert_eq!(policy.workspace_policy(), &WorkspacePolicy::Ephemeral);
10469 harness.policy = policy;
10470 harness.ready().await;
10471
10472 let attempt = harness.launch().await;
10473 assert_eq!(attempt.workspace(), AttemptWorkspace::Ephemeral);
10474 assert_eq!(attempt.workspace().slot_number(), None);
10475 assert!(!attempt.holds_slot_lease());
10476 assert_eq!(
10477 attempt.runtime_path().parent().unwrap(),
10478 harness.host_root(),
10479 "a disposable attempt is a child of the host root, never of a slot"
10480 );
10481 assert!(
10482 harness
10483 .store
10484 .slot_leases_for_policy(harness.policy.id)
10485 .unwrap()
10486 .is_empty()
10487 );
10488 }
10489 }
10490
10491 #[tokio::test]
10492 async fn two_concurrent_allocations_never_share_a_slot() {
10493 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10494 .with_persistent_workspace(2);
10495 harness.github.watch_journal(Arc::clone(&harness.store));
10496 harness.ready().await;
10497
10498 let (first, second) = tokio::join!(harness.launch_result(), harness.launch_result());
10502 let first = first.unwrap();
10503 let second = second.unwrap();
10504
10505 let slots: BTreeSet<u16> = [&first, &second]
10506 .iter()
10507 .map(|attempt| {
10508 attempt
10509 .workspace()
10510 .slot_number()
10511 .expect("a persistent attempt leases a slot")
10512 })
10513 .collect();
10514 assert_eq!(slots, BTreeSet::from([1, 2]), "one slot each, never shared");
10515 assert_ne!(first.runtime_path(), second.runtime_path());
10516 assert_eq!(
10517 harness
10518 .store
10519 .slot_leases_for_policy(harness.policy.id)
10520 .unwrap()
10521 .len(),
10522 2
10523 );
10524
10525 let facts = harness.github.registration_facts();
10530 assert_eq!(facts.len(), 2);
10531 for fact in facts {
10532 let attempt = [&first, &second]
10533 .into_iter()
10534 .find(|attempt| runner_name(attempt.id) == fact.runner_name)
10535 .expect("every registration belongs to one of the two attempts");
10536 let slot = attempt
10537 .workspace()
10538 .slot_number()
10539 .expect("a persistent attempt leases a slot");
10540 assert!(
10541 fact.leased_slots.contains(&slot),
10542 "a JIT request never precedes its own lease: s{slot} not in {:?}",
10543 fact.leased_slots
10544 );
10545 assert_eq!(fact.work_folder, DEFAULT_WORK_FOLDER);
10546 }
10547 }
10548
10549 #[tokio::test]
10550 async fn the_database_is_the_final_fence_against_two_attempts_in_one_slot() {
10551 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10552 .with_persistent_workspace(2);
10553 harness.ready().await;
10554 let first = harness.launch().await;
10555
10556 let clash = RunnerAttempt::allocate_in(
10560 AttemptId::new_random(),
10561 harness.policy.id,
10562 first.runtime_path(),
10563 AttemptWorkspace::persistent_slot(nz(1)),
10564 harness.clock.now(),
10565 );
10566 assert!(matches!(
10567 harness.store.record_attempt(&clash).unwrap_err(),
10568 StoreError::SlotAlreadyLeased { slot: 1, .. }
10569 ));
10570
10571 let error = harness.launcher.record_allocation(&clash).unwrap_err();
10574 let rendered = error.to_string();
10575 assert!(rendered.contains("slot s1"), "{rendered}");
10576 assert!(rendered.contains("nothing was written"), "{rendered}");
10577 assert_eq!(
10578 harness.store.attempts().unwrap().len(),
10579 1,
10580 "the losing allocator journalled nothing"
10581 );
10582 }
10583
10584 #[test]
10585 fn slot_selection_fills_the_lowest_gap_and_stops_at_the_ceiling() {
10586 let leased = |slots: &[u16]| -> Vec<RunnerAttempt> {
10587 slots
10588 .iter()
10589 .map(|slot| {
10590 RunnerAttempt::allocate_in(
10591 AttemptId::new_random(),
10592 fixtures::POLICY_ID,
10593 format!("/srv/rman/acme/s{slot}"),
10594 AttemptWorkspace::persistent_slot(nz(*slot)),
10595 fixtures::created_at(),
10596 )
10597 })
10598 .collect()
10599 };
10600
10601 assert_eq!(lowest_free_slot(&[], nz(1)), Some(nz(1)));
10602 assert_eq!(lowest_free_slot(&leased(&[1]), nz(4)), Some(nz(2)));
10603 assert_eq!(lowest_free_slot(&leased(&[1, 3]), nz(4)), Some(nz(2)));
10605 assert_eq!(lowest_free_slot(&leased(&[1]), nz(1)), None);
10607 assert_eq!(lowest_free_slot(&leased(&[1, 2]), nz(2)), None);
10608 let ephemeral = vec![RunnerAttempt::allocate(
10610 AttemptId::new_random(),
10611 fixtures::POLICY_ID,
10612 "/srv/rman/host/abc",
10613 fixtures::created_at(),
10614 )];
10615 assert_eq!(lowest_free_slot(&ephemeral, nz(1)), Some(nz(1)));
10616 }
10617
10618 #[test]
10619 fn a_slot_is_reusable_only_when_it_is_empty_or_holds_one_real_work_directory() {
10620 let root = tempfile::tempdir().unwrap();
10621 let slot = root.path().join("s1");
10622 fs::create_dir(&slot).unwrap();
10623 accept_reusable_slot(&slot).expect("an empty slot is reusable");
10624
10625 fs::create_dir(slot.join(DEFAULT_WORK_FOLDER)).unwrap();
10626 accept_reusable_slot(&slot).expect("a retained job workspace is reusable");
10627
10628 fs::create_dir(slot.join("bin")).unwrap();
10631 fs::write(slot.join(".github-runner-id"), b"73").unwrap();
10632 let refusal = accept_reusable_slot(&slot).unwrap_err().to_string();
10633 assert!(refusal.contains("bin"), "{refusal}");
10634 assert!(refusal.contains(".github-runner-id"), "{refusal}");
10635
10636 let file_work = root.path().join("s2");
10638 fs::create_dir(&file_work).unwrap();
10639 fs::write(file_work.join(DEFAULT_WORK_FOLDER), b"not a directory").unwrap();
10640 assert!(accept_reusable_slot(&file_work).is_err());
10641 }
10642
10643 #[cfg(unix)]
10644 #[test]
10645 fn a_link_shaped_work_directory_is_refused_rather_than_followed() {
10646 let root = tempfile::tempdir().unwrap();
10650 let elsewhere = root.path().join("elsewhere");
10651 fs::create_dir(&elsewhere).unwrap();
10652
10653 let slot = root.path().join("s1");
10654 fs::create_dir(&slot).unwrap();
10655 std::os::unix::fs::symlink(&elsewhere, slot.join(DEFAULT_WORK_FOLDER)).unwrap();
10656 assert!(accept_reusable_slot(&slot).is_err());
10657
10658 let linked_slot = root.path().join("s2");
10659 std::os::unix::fs::symlink(&elsewhere, &linked_slot).unwrap();
10660 assert!(create_or_validate_slot(&linked_slot).is_err());
10661 }
10662
10663 #[test]
10664 fn a_slot_standing_where_a_file_is_refuses_rather_than_replacing_it() {
10665 let root = tempfile::tempdir().unwrap();
10666 let occupied = root.path().join("s1");
10667 fs::write(&occupied, b"an operator's file").unwrap();
10668 let refusal = create_or_validate_slot(&occupied).unwrap_err().to_string();
10669 assert!(refusal.contains("is not a directory"), "{refusal}");
10670 assert_eq!(fs::read_to_string(&occupied).unwrap(), "an operator's file");
10671
10672 let fresh = root.path().join("s2");
10673 create_or_validate_slot(&fresh).expect("a missing slot is created");
10674 assert!(fresh.is_dir());
10675 create_or_validate_slot(&fresh).expect("an existing directory is accepted");
10676 }
10677
10678 #[test]
10679 fn the_retained_work_directory_is_matched_the_way_the_filesystem_matches_it() {
10680 assert!(is_work_folder(OsStr::new(DEFAULT_WORK_FOLDER)));
10681 assert!(!is_work_folder(OsStr::new("_work2")));
10682 assert_eq!(is_work_folder(OsStr::new("_Work")), cfg!(windows));
10686 }
10687
10688 #[test]
10689 fn package_materialization_never_overwrites_or_follows_a_retained_work_directory() {
10690 let root = tempfile::tempdir().unwrap();
10691 let package = root.path().join("package");
10692 fs::create_dir_all(package.join("bin")).unwrap();
10693 fs::write(package.join("bin").join("Runner.Listener"), b"binary").unwrap();
10694 fs::create_dir_all(package.join("externals").join(DEFAULT_WORK_FOLDER)).unwrap();
10696
10697 let slot = root.path().join("s1");
10698 let retained = slot.join(DEFAULT_WORK_FOLDER).join("repo");
10699 fs::create_dir_all(&retained).unwrap();
10700 fs::write(retained.join("checkout.txt"), b"from the first job").unwrap();
10701
10702 copy_package_tree(&package, &slot, &TrustedTrees::default())
10703 .expect("the package lays out around `_work`");
10704 assert!(slot.join("bin").join("Runner.Listener").exists());
10705 assert!(
10706 slot.join("externals").join(DEFAULT_WORK_FOLDER).is_dir(),
10707 "the guard is top-level only"
10708 );
10709 assert_eq!(
10710 fs::read_to_string(retained.join("checkout.txt")).unwrap(),
10711 "from the first job"
10712 );
10713
10714 fs::create_dir(package.join(DEFAULT_WORK_FOLDER)).unwrap();
10716 let error = copy_package_tree(&package, &slot, &TrustedTrees::default()).unwrap_err();
10717 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
10718 assert_eq!(
10719 fs::read_to_string(retained.join("checkout.txt")).unwrap(),
10720 "from the first job"
10721 );
10722 }
10723
10724 #[test]
10725 fn rolling_back_a_materialization_keeps_a_slot_but_removes_a_disposable_directory() {
10726 let root = tempfile::tempdir().unwrap();
10727
10728 let slot = root.path().join("s1");
10729 let retained = slot.join(DEFAULT_WORK_FOLDER);
10730 fs::create_dir_all(retained.join("repo")).unwrap();
10731 fs::write(retained.join("repo").join("checkout.txt"), b"kept").unwrap();
10732 fs::create_dir_all(slot.join("bin")).unwrap();
10733 fs::write(slot.join(".github-runner-id"), b"73").unwrap();
10734 let persistent = RunnerAttempt::allocate_in(
10735 AttemptId::new_random(),
10736 fixtures::POLICY_ID,
10737 &slot,
10738 AttemptWorkspace::persistent_slot(nz(1)),
10739 fixtures::created_at(),
10740 );
10741
10742 remove_materialized_package(&persistent).unwrap();
10743 assert!(slot.is_dir(), "the slot itself is not removed");
10744 assert!(!slot.join("bin").exists());
10745 assert!(!slot.join(".github-runner-id").exists());
10746 assert_eq!(
10747 fs::read_to_string(retained.join("repo").join("checkout.txt")).unwrap(),
10748 "kept"
10749 );
10750
10751 let disposable_path = root.path().join("abcdef012345");
10752 fs::create_dir_all(disposable_path.join(DEFAULT_WORK_FOLDER)).unwrap();
10753 let disposable = RunnerAttempt::allocate(
10754 AttemptId::new_random(),
10755 fixtures::POLICY_ID,
10756 &disposable_path,
10757 fixtures::created_at(),
10758 );
10759 remove_materialized_package(&disposable).unwrap();
10760 assert!(
10761 !disposable_path.exists(),
10762 "a disposable directory is still removed whole"
10763 );
10764 }
10765
10766 fn litter_the_slot(slot: &Path) {
10776 for directory in ["bin", "externals", "_diag"] {
10777 fs::create_dir_all(slot.join(directory)).unwrap();
10778 }
10779 fs::write(slot.join("bin").join("Runner.Listener"), b"binary").unwrap();
10780 for file in SENSITIVE_SLOT_ENTRIES
10781 .iter()
10782 .filter(|entry| !slot.join(entry).is_dir())
10783 {
10784 fs::write(slot.join(file), b"runner state").unwrap();
10785 }
10786 fs::write(
10788 slot.join(format!(
10789 "{}0f1e2d3c-4b5a-6978-8796-a5b4c3d2e1f0.tmp",
10790 RestrictiveHandoff::NAME_PREFIX
10791 )),
10792 JIT.as_bytes(),
10793 )
10794 .unwrap();
10795 }
10796
10797 fn retain_under_work(slot: &Path) -> PathBuf {
10799 let checkout = slot.join(DEFAULT_WORK_FOLDER).join("repo").join("target");
10800 fs::create_dir_all(&checkout).unwrap();
10801 let marker = checkout.join("build-output.bin");
10802 fs::write(&marker, RETAINED).unwrap();
10803 marker
10804 }
10805
10806 const RETAINED: &str = "a Git-ignored build output the next job reuses";
10808
10809 fn entries_of(directory: &Path) -> Vec<String> {
10811 let mut names: Vec<String> = fs::read_dir(directory)
10812 .unwrap()
10813 .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
10814 .collect();
10815 names.sort();
10816 names
10817 }
10818
10819 fn only_the_job_workspace() -> Vec<String> {
10821 vec![DEFAULT_WORK_FOLDER.to_owned()]
10822 }
10823
10824 #[cfg(unix)]
10825 #[test]
10826 fn disposable_tree_removal_does_not_open_a_dotnet_diagnostic_fifo() {
10827 use std::sync::mpsc;
10828
10829 let temporary = tempfile::tempdir().unwrap();
10830 let tree = temporary.path().join("attempt");
10831 let diagnostic = tree.join("tmp/clr-debug-pipe-runner-in");
10832 fs::create_dir_all(diagnostic.parent().unwrap()).unwrap();
10833 assert!(
10834 std::process::Command::new("mkfifo")
10835 .arg(&diagnostic)
10836 .status()
10837 .unwrap()
10838 .success()
10839 );
10840
10841 let (finished, result) = mpsc::channel();
10842 std::thread::spawn(move || {
10843 let removed = remove_runtime_tree(&tree);
10844 let _ = finished.send(removed);
10845 });
10846
10847 result
10848 .recv_timeout(Duration::from_secs(2))
10849 .expect("runtime deletion must not wait for a FIFO peer")
10850 .unwrap();
10851 assert!(!diagnostic.exists());
10852 }
10853
10854 #[cfg(windows)]
10857 #[test]
10858 fn disposable_tree_removal_unlinks_a_junction_nobody_may_list() {
10859 let temporary = tempfile::tempdir().unwrap();
10860 let tree = temporary.path().join("attempt");
10861 let cache = tree.join(r"tmp\home\AppData\Local\Microsoft\Windows\INetCache");
10862 let outside = temporary.path().join("outside");
10863 fs::create_dir_all(&cache).unwrap();
10864 fs::create_dir_all(&outside).unwrap();
10865 fs::write(outside.join("keep.txt"), b"not the attempt's").unwrap();
10866 fs::write(tree.join("tmp").join("job.log"), b"job output").unwrap();
10867 let junction = cache.join("Content.IE5");
10868 let created = std::process::Command::new("cmd")
10869 .arg("/C")
10870 .arg("mklink")
10871 .arg("/J")
10872 .arg(&junction)
10873 .arg(&outside)
10874 .output()
10875 .unwrap();
10876 assert!(created.status.success(), "{created:?}");
10877 let denied = std::process::Command::new("icacls")
10881 .arg(&junction)
10882 .args(["/L", "/deny", "*S-1-1-0:(RD)"])
10883 .output()
10884 .unwrap();
10885 assert!(denied.status.success(), "{denied:?}");
10886
10887 let unfixed = remove_dir_all::remove_dir_all(&tree)
10889 .expect_err("the fixture no longer reproduces the WinINet junction");
10890 assert_eq!(unfixed.kind(), std::io::ErrorKind::PermissionDenied);
10891 assert!(junction.symlink_metadata().is_ok());
10892
10893 remove_runtime_tree(&tree).expect("a tree holding a deny-list junction is removed");
10894 assert!(!tree.exists(), "the attempt tree survived");
10895 assert_eq!(
10896 fs::read(outside.join("keep.txt")).unwrap(),
10897 b"not the attempt's",
10898 "removing the junction followed it into the directory it points at"
10899 );
10900 }
10901
10902 struct BlockedDeletion {
10918 directory: PathBuf,
10919 #[cfg(windows)]
10920 _handle: fs::File,
10921 }
10922
10923 impl BlockedDeletion {
10924 const HELD: &'static str = "held-open";
10925
10926 fn inject(directory: &Path) -> Option<Self> {
10929 #[cfg(unix)]
10930 if !Self::refusal_is_possible() {
10931 return None;
10932 }
10933 fs::create_dir_all(directory).unwrap();
10934 fs::write(
10935 directory.join(Self::HELD),
10936 b"a file the scrub cannot remove",
10937 )
10938 .unwrap();
10939 #[cfg(windows)]
10940 let handle = {
10941 use std::os::windows::fs::OpenOptionsExt;
10942
10943 fs::OpenOptions::new()
10944 .read(true)
10945 .share_mode(0)
10946 .open(directory.join(Self::HELD))
10947 .expect("the blocking handle opens")
10948 };
10949 #[cfg(unix)]
10950 Self::set_mode(directory, 0o555);
10951 Some(Self {
10952 directory: directory.to_path_buf(),
10953 #[cfg(windows)]
10954 _handle: handle,
10955 })
10956 }
10957
10958 fn release(self) {
10959 drop(self);
10960 }
10961
10962 #[cfg(unix)]
10963 fn refusal_is_possible() -> bool {
10964 let probe = tempfile::tempdir().unwrap();
10965 let directory = probe.path().join("probe");
10966 fs::create_dir(&directory).unwrap();
10967 fs::write(directory.join("file"), b"probe").unwrap();
10968 Self::set_mode(&directory, 0o555);
10969 let refused = fs::remove_dir_all(&directory).is_err();
10970 Self::set_mode(&directory, 0o755);
10971 refused
10972 }
10973
10974 #[cfg(unix)]
10975 fn set_mode(directory: &Path, mode: u32) {
10976 use std::os::unix::fs::PermissionsExt;
10977
10978 let mut permissions = fs::metadata(directory).unwrap().permissions();
10979 permissions.set_mode(mode);
10980 fs::set_permissions(directory, permissions).unwrap();
10981 }
10982 }
10983
10984 impl Drop for BlockedDeletion {
10985 fn drop(&mut self) {
10986 #[cfg(unix)]
10987 Self::set_mode(&self.directory, 0o755);
10988 #[cfg(not(unix))]
10989 let _ = &self.directory;
10990 }
10991 }
10992
10993 #[tokio::test]
10994 async fn two_sequential_jobs_keep_the_checkout_and_start_without_the_earlier_runner_state() {
10995 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
10996 .with_persistent_workspace(1);
10997 harness.ready().await;
10998
10999 let first = harness.launch().await;
11000 let slot = harness.slot_path(1);
11001 assert_eq!(first.runtime_path(), slot);
11002 assert_eq!(
11003 read_runner_id(&slot),
11004 Some(73),
11005 "the attempt registered, so its identity is on disk"
11006 );
11007 let marker = retain_under_work(&slot);
11008 litter_the_slot(&slot);
11009
11010 harness.cleanup_retaining_work(first.id).await;
11011
11012 assert_eq!(entries_of(&slot), only_the_job_workspace());
11015 assert_eq!(fs::read_to_string(&marker).unwrap(), RETAINED);
11016 assert_eq!(
11017 read_runner_id(&slot),
11018 None,
11019 "the first attempt's registration identity is gone before the second starts"
11020 );
11021 assert_eq!(harness.attempt(first.id).state(), AttemptState::Cleaned);
11022 assert!(!harness.attempt(first.id).holds_slot_lease());
11023
11024 let second = harness.launch().await;
11025 assert_ne!(second.id, first.id);
11026 assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(1)));
11027 assert_eq!(
11028 second.runtime_path(),
11029 slot,
11030 "the same slot, so the same retained `_work`"
11031 );
11032 assert_eq!(fs::read_to_string(&marker).unwrap(), RETAINED);
11033 }
11034
11035 #[tokio::test]
11036 async fn cleaning_a_persistent_slot_needs_no_policy_and_scans_no_directory_for_ownership() {
11037 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11038 .with_persistent_workspace(1);
11039 harness.ready().await;
11040 let attempt = harness.launch().await;
11041 let slot = harness.slot_path(1);
11042 let marker = retain_under_work(&slot);
11043 litter_the_slot(&slot);
11044 harness.conclude(attempt.id);
11045
11046 harness
11052 .store
11053 .remove_policy(harness.policy.id, harness.policy.revision())
11054 .unwrap();
11055 assert!(harness.store.policy(harness.policy.id).unwrap().is_none());
11056
11057 harness
11058 .launcher
11059 .clean(attempt.id)
11060 .await
11061 .expect("journal facts alone are enough to clean the slot");
11062
11063 assert_eq!(entries_of(&slot), only_the_job_workspace());
11064 assert!(marker.exists());
11065 assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
11066 }
11067
11068 #[tokio::test]
11069 async fn an_injected_partial_deletion_quarantines_the_slot_across_a_restart() {
11070 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11071 .with_persistent_workspace(2);
11072 harness.ready().await;
11073 let first = harness.launch().await;
11074 let slot = harness.slot_path(1);
11075 let marker = retain_under_work(&slot);
11076 litter_the_slot(&slot);
11077 harness.conclude(first.id);
11078
11079 let Some(block) = BlockedDeletion::inject(&slot.join("bin")) else {
11080 eprintln!(
11081 "skipped: this account cannot be refused a deletion, so no partial deletion can \
11082 be injected"
11083 );
11084 return;
11085 };
11086
11087 let refusal = harness
11088 .launcher
11089 .clean(first.id)
11090 .await
11091 .expect_err("a deletion that failed may not report a cleaned slot");
11092 let rendered = refusal.reason.to_string();
11093 assert!(rendered.contains("could not be removed"), "{rendered}");
11094
11095 let held = harness.attempt(first.id);
11096 assert_eq!(held.state(), AttemptState::Failed, "still not cleaned");
11097 assert!(held.holds_slot_lease(), "so the slot is still leased");
11098 assert!(
11099 !held.state().counts_against_capacity(),
11100 "and a concluded attempt still costs the host no capacity"
11101 );
11102
11103 let restarted = harness.restart();
11108 restarted
11109 .recover_startup(std::slice::from_ref(&harness.policy))
11110 .await
11111 .expect("one quarantined slot does not stop the host recovering");
11112 assert_eq!(
11113 harness.attempt(first.id).state(),
11114 AttemptState::Failed,
11115 "the quarantine survived the restart"
11116 );
11117 assert!(
11118 harness
11119 .reconcile_events
11120 .events()
11121 .iter()
11122 .any(|event| matches!(
11123 event,
11124 LifecycleEvent::AttemptCleanFailed {
11125 reason: "slot_entry_could_not_be_removed",
11126 ..
11127 }
11128 )),
11129 "the refusal is reported rather than retried in silence"
11130 );
11131
11132 let second = harness
11135 .launch_through(&restarted)
11136 .await
11137 .expect("the host can still launch");
11138 assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
11139
11140 block.release();
11143 restarted
11144 .clean(first.id)
11145 .await
11146 .expect("the retried cleanup completes");
11147 assert_eq!(entries_of(&slot), only_the_job_workspace());
11148 assert!(marker.exists());
11149 assert_eq!(harness.attempt(first.id).state(), AttemptState::Cleaned);
11150 }
11151
11152 #[tokio::test]
11163 async fn an_injected_deletion_failure_leaves_a_disposable_attempt_uncleaned() {
11164 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
11165 harness.ready().await;
11166 let attempt = harness.launch().await;
11167 let runtime = attempt.runtime_path().to_path_buf();
11168 assert_eq!(attempt.workspace(), AttemptWorkspace::Ephemeral);
11169 harness.conclude(attempt.id);
11170
11171 let Some(block) = BlockedDeletion::inject(&runtime.join("held-open-subdirectory")) else {
11172 eprintln!(
11173 "skipped: this account cannot be refused a deletion, so no partial deletion can be injected"
11174 );
11175 return;
11176 };
11177
11178 let refusal = harness
11179 .launcher
11180 .clean(attempt.id)
11181 .await
11182 .expect_err("a deletion that failed may not report a removed workspace");
11183 let rendered = refusal.reason.to_string();
11184 assert!(
11185 rendered.contains("could not be removed"),
11186 "the refusal names what happened: {rendered}"
11187 );
11188 assert_ne!(
11189 harness.attempt(attempt.id).state(),
11190 AttemptState::Cleaned,
11191 "an attempt whose directory is still on disk is not cleaned"
11192 );
11193 assert!(
11194 runtime.is_dir(),
11195 "the directory the removal could not finish is still there, which is the fact the journal must keep agreeing with"
11196 );
11197
11198 let restarted = harness.restart();
11199 restarted
11200 .recover_startup(std::slice::from_ref(&harness.policy))
11201 .await
11202 .expect("one locked ephemeral workspace does not stop startup recovery");
11203 assert!(
11204 harness
11205 .reconcile_events
11206 .events()
11207 .iter()
11208 .any(|event| matches!(
11209 event,
11210 LifecycleEvent::AttemptCleanFailed {
11211 attempt: failed_attempt,
11212 reason: "ephemeral_workspace_could_not_be_removed",
11213 ..
11214 } if *failed_attempt == attempt.id
11215 )),
11216 "the deferred cleanup remains visible while the daemon keeps running"
11217 );
11218
11219 block.release();
11222 harness
11223 .launcher
11224 .clean(attempt.id)
11225 .await
11226 .expect("the retried cleanup completes");
11227 assert!(!runtime.exists(), "the whole attempt directory goes");
11228 assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
11229 }
11230
11231 #[tokio::test]
11235 async fn supervision_backs_off_a_workspace_it_cannot_remove() {
11236 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand));
11237 harness.ready().await;
11238 let attempt = harness.launch().await;
11239 let runtime = attempt.runtime_path().to_path_buf();
11240 harness.conclude(attempt.id);
11241 let Some(block) = BlockedDeletion::inject(&runtime.join("held-open-subdirectory")) else {
11242 eprintln!(
11243 "skipped: this account cannot be refused a deletion, so no partial deletion can be injected"
11244 );
11245 return;
11246 };
11247 let failures = || {
11248 harness
11249 .reconcile_events
11250 .events()
11251 .iter()
11252 .filter(|event| {
11253 matches!(
11254 event,
11255 LifecycleEvent::AttemptCleanFailed { attempt: failed, .. }
11256 if *failed == attempt.id
11257 )
11258 })
11259 .count()
11260 };
11261 let supervise = async || {
11262 harness
11263 .launcher
11264 .supervise(&harness.policy)
11265 .await
11266 .expect("a stuck workspace does not fail supervision");
11267 };
11268
11269 supervise().await;
11270 assert_eq!(failures(), 1);
11271 supervise().await;
11272 assert_eq!(
11273 failures(),
11274 1,
11275 "the failed cleanup was retried on the very next pass"
11276 );
11277 harness.clock.advance_secs(31);
11278 supervise().await;
11279 assert_eq!(
11280 failures(),
11281 2,
11282 "the cleanup was not retried once its delay passed"
11283 );
11284
11285 block.release();
11286 harness.clock.advance_secs(61);
11287 supervise().await;
11288 assert!(
11289 !runtime.exists(),
11290 "the retry removes the workspace once it can"
11291 );
11292 assert_eq!(harness.attempt(attempt.id).state(), AttemptState::Cleaned);
11293 }
11294
11295 #[tokio::test]
11296 async fn changing_a_repository_back_to_ephemeral_leaves_every_old_slot_untouched() {
11297 let mut harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11298 .with_persistent_workspace(1);
11299 harness.ready().await;
11300 let first = harness.launch().await;
11301 let slot = harness.slot_path(1);
11302 let marker = retain_under_work(&slot);
11303 harness.cleanup_retaining_work(first.id).await;
11304
11305 harness
11309 .policy
11310 .set_workspace_policy(WorkspacePolicy::Ephemeral)
11311 .unwrap();
11312
11313 let second = harness.launch().await;
11314 assert_eq!(second.workspace(), AttemptWorkspace::Ephemeral);
11315 assert_eq!(
11316 second.runtime_path().parent().unwrap(),
11317 harness.host_root(),
11318 "a disposable attempt is a child of the host root"
11319 );
11320 assert!(slot.is_dir(), "the old slot is left where it stands");
11321 assert_eq!(fs::read_to_string(&marker).unwrap(), RETAINED);
11322
11323 harness.conclude(second.id);
11326 harness.launcher.clean(second.id).await.unwrap();
11327 assert!(!second.runtime_path().exists());
11328 assert!(marker.exists());
11329 }
11330
11331 #[tokio::test]
11332 async fn a_persistent_slot_is_scrubbed_only_after_the_process_is_signalled_and_gone() {
11333 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11334 .with_persistent_workspace(1);
11335 harness.ready().await;
11336 let slot = harness.slot_path(1);
11337 fs::create_dir_all(&slot).unwrap();
11338 let marker = retain_under_work(&slot);
11339 litter_the_slot(&slot);
11340
11341 let id = AttemptId::new_random();
11342 let mut attempt = RunnerAttempt::allocate_in(
11343 id,
11344 harness.policy.id,
11345 &slot,
11346 AttemptWorkspace::persistent_slot(nz(1)),
11347 harness.clock.now(),
11348 );
11349 attempt.jit_received(harness.clock.now()).unwrap();
11350 attempt.started(4242, harness.clock.now()).unwrap();
11351 harness.store.record_attempt(&attempt).unwrap();
11352 harness.clock.advance_secs(11);
11353 harness.processes.set_alive(true);
11354 harness
11355 .github
11356 .observe(GithubRunnerObservation::NotRegistered);
11357
11358 harness.launcher.supervise(&harness.policy).await.unwrap();
11359
11360 let actions = harness.processes.actions.lock().unwrap().clone();
11364 let intent = actions
11365 .iter()
11366 .position(|action| *action == "terminate_intent")
11367 .unwrap();
11368 let signal = actions
11369 .iter()
11370 .position(|action| *action == "terminate")
11371 .unwrap();
11372 assert!(intent < signal, "{actions:?}");
11373 assert!(!harness.processes.alive.load(Ordering::SeqCst));
11374
11375 let cleaned = harness.attempt(id);
11376 assert_eq!(cleaned.state(), AttemptState::Cleaned);
11377 assert!(matches!(
11378 cleaned.outcome(),
11379 Some(AttemptOutcome::Failed {
11380 reason: FailureReason::TerminatedAfterRegistrationTimeout
11381 })
11382 ));
11383 assert_eq!(entries_of(&slot), only_the_job_workspace());
11384 assert!(marker.exists());
11385 }
11386
11387 #[test]
11388 fn a_scrub_retains_one_real_work_directory_and_removes_every_other_entry() {
11389 let root = tempfile::tempdir().unwrap();
11390 let slot = root.path().join("s1");
11391 fs::create_dir(&slot).unwrap();
11392 let marker = retain_under_work(&slot);
11393 litter_the_slot(&slot);
11394 fs::write(slot.join("runner-package"), b"verified").unwrap();
11395
11396 scrub_slot_entries(&slot).expect("a slot of ordinary runner state scrubs");
11397 verify_slot_scrubbed(&slot).expect("and proves it afterwards");
11398
11399 assert_eq!(entries_of(&slot), only_the_job_workspace());
11400 assert!(marker.exists());
11401 }
11402
11403 #[test]
11404 fn a_residue_refusal_never_reports_the_under_count_as_the_fact() {
11405 let slot = Path::new("/runners/s1");
11406
11407 let counted = residue_detail(slot, 2, &["`bin`".to_owned()]);
11410 assert!(counted.contains("2 entries other than"), "{counted}");
11411 assert!(counted.contains("including `bin`"), "{counted}");
11412 assert_eq!(
11413 residue_detail(slot, 1, &[]),
11414 format!(
11415 "1 entry other than `{DEFAULT_WORK_FOLDER}` survived cleanup of {}",
11416 slot.display()
11417 )
11418 );
11419
11420 let raced = residue_detail(slot, 0, &["`.credentials`".to_owned()]);
11425 assert!(!raced.contains('0'), "{raced}");
11426 assert!(raced.contains("reported nothing but"), "{raced}");
11427 assert!(raced.contains("`.credentials` survived cleanup"), "{raced}");
11428 }
11429
11430 #[test]
11431 fn verification_asks_the_filesystem_rather_than_the_listing_that_missed_an_entry() {
11432 let root = tempfile::tempdir().unwrap();
11433 let slot = root.path().join("s1");
11434 fs::create_dir(&slot).unwrap();
11435 fs::create_dir(slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11436 verify_slot_scrubbed(&slot).expect("only `_work` is a clean slot");
11437
11438 for survivor in ["bin", ".credentials", IDENTITY_FILE, RUNNER_ID_FILE] {
11442 fs::write(slot.join(survivor), b"left behind").unwrap();
11443 let quarantine = verify_slot_scrubbed(&slot).unwrap_err();
11444 assert_eq!(quarantine.refusal, SlotRefusal::Residue);
11445 assert!(
11446 quarantine.detail.contains(&format!("`{survivor}`")),
11447 "{quarantine}"
11448 );
11449 fs::remove_file(slot.join(survivor)).unwrap();
11450 }
11451
11452 let handoff = slot.join(format!("{}whatever.tmp", RestrictiveHandoff::NAME_PREFIX));
11455 fs::write(&handoff, JIT.as_bytes()).unwrap();
11456 let quarantine = verify_slot_scrubbed(&slot).unwrap_err();
11457 assert!(
11458 quarantine.detail.contains("an encoded JIT handoff"),
11459 "{quarantine}"
11460 );
11461 assert!(!quarantine.detail.contains(JIT), "{quarantine}");
11462 fs::remove_file(&handoff).unwrap();
11463
11464 fs::write(slot.join("ghp_DO_NOT_LEAK"), b"named by the job").unwrap();
11468 let quarantine = verify_slot_scrubbed(&slot).unwrap_err();
11469 assert!(
11470 quarantine.detail.contains("1 entry other than"),
11471 "{quarantine}"
11472 );
11473 assert!(
11474 !quarantine.detail.contains("ghp_DO_NOT_LEAK"),
11475 "{quarantine}"
11476 );
11477 }
11478
11479 #[test]
11480 fn a_slot_is_derived_from_the_journal_and_refused_when_it_disagrees() {
11481 let root = tempfile::tempdir().unwrap();
11482 let configured =
11483 LocalAbsolutePath::new(root.path().to_str().unwrap()).expect("a local absolute root");
11484 let slot = configured.as_path().join("s1");
11485 fs::create_dir(&slot).unwrap();
11486
11487 verify_journalled_slot(&slot, nz(1), Some(&configured))
11488 .expect("the journalled slot agrees");
11489 verify_journalled_slot(&slot, nz(1), None)
11490 .expect("and a policy that is gone removes a check, not the ability to clean");
11491
11492 assert_eq!(
11495 verify_journalled_slot(&slot, nz(2), None)
11496 .unwrap_err()
11497 .refusal,
11498 SlotRefusal::NotTheJournalledSlot
11499 );
11500 for stray in ["s1/nested", "not-a-slot", "s01"] {
11501 let path = configured.as_path().join(stray);
11502 assert_eq!(
11503 verify_journalled_slot(&path, nz(1), None)
11504 .unwrap_err()
11505 .refusal,
11506 SlotRefusal::NotTheJournalledSlot,
11507 "{}",
11508 path.display()
11509 );
11510 }
11511
11512 let elsewhere = tempfile::tempdir().unwrap();
11515 let other =
11516 LocalAbsolutePath::new(elsewhere.path().to_str().unwrap()).expect("a second root");
11517 assert_eq!(
11518 verify_journalled_slot(&slot, nz(1), Some(&other))
11519 .unwrap_err()
11520 .refusal,
11521 SlotRefusal::PolicyRootDisagrees
11522 );
11523 }
11524
11525 #[cfg(unix)]
11526 #[test]
11527 fn a_substituted_work_directory_quarantines_the_slot_and_deletes_nothing_outside_it() {
11528 let root = tempfile::tempdir().unwrap();
11532 let outside = root.path().join("operator-data");
11533 fs::create_dir(&outside).unwrap();
11534 let sentinel = outside.join("do-not-delete.txt");
11535 fs::write(
11536 &sentinel,
11537 b"an operator's data, outside every approved root",
11538 )
11539 .unwrap();
11540
11541 let slot = root.path().join("s1");
11542 fs::create_dir(&slot).unwrap();
11543 fs::create_dir(slot.join("bin")).unwrap();
11544 std::os::unix::fs::symlink(&outside, slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11545
11546 let quarantine = scrub_slot_entries(&slot).unwrap_err();
11547 assert_eq!(quarantine.refusal, SlotRefusal::WorkNotADirectory);
11548 assert!(
11549 sentinel.exists(),
11550 "the deletion followed the link out of the slot"
11551 );
11552 assert!(outside.is_dir());
11553 assert!(
11554 slot.join(DEFAULT_WORK_FOLDER).symlink_metadata().is_ok(),
11555 "the substituted link is left for the operator, never unlinked as if it were ours"
11556 );
11557
11558 let file_work = root.path().join("s2");
11562 fs::create_dir(&file_work).unwrap();
11563 fs::write(file_work.join(DEFAULT_WORK_FOLDER), b"not a directory").unwrap();
11564 assert_eq!(
11565 scrub_slot_entries(&file_work).unwrap_err().refusal,
11566 SlotRefusal::WorkNotADirectory
11567 );
11568 }
11569
11570 #[cfg(unix)]
11571 #[test]
11572 fn a_slot_replaced_by_a_link_out_of_its_root_is_refused_before_anything_is_read() {
11573 let root = tempfile::tempdir().unwrap();
11574 let outside = root.path().join("operator-data");
11575 fs::create_dir(&outside).unwrap();
11576 let sentinel = outside.join("do-not-delete.txt");
11577 fs::write(
11578 &sentinel,
11579 b"an operator's data, outside every approved root",
11580 )
11581 .unwrap();
11582
11583 let inside = root.path().join("inside");
11586 fs::create_dir(&inside).unwrap();
11587 let slot = inside.join("s1");
11588 std::os::unix::fs::symlink(&outside, &slot).unwrap();
11589
11590 assert_eq!(
11591 verify_journalled_slot(&slot, nz(1), None)
11592 .unwrap_err()
11593 .refusal,
11594 SlotRefusal::Containment
11595 );
11596 assert!(sentinel.exists());
11597 assert!(
11598 slot.symlink_metadata().is_ok(),
11599 "the link is left for the operator rather than removed as if it were ours"
11600 );
11601 }
11602
11603 #[cfg(windows)]
11613 #[test]
11614 fn a_slot_root_replaced_by_a_junction_is_refused_before_anything_is_read() {
11615 let root = tempfile::tempdir().unwrap();
11616 let outside = root.path().join("operator-data");
11617 fs::create_dir(&outside).unwrap();
11618 let sentinel = outside.join("do-not-delete.txt");
11619 fs::write(
11620 &sentinel,
11621 b"an operator's data, outside every approved root",
11622 )
11623 .unwrap();
11624
11625 let inside = root.path().join("inside");
11628 fs::create_dir(&inside).unwrap();
11629 let slot = inside.join("s1");
11630 let Some(()) = plant_junction(&slot, &outside) else {
11631 eprintln!("skipped: this machine would not create a directory junction");
11632 return;
11633 };
11634
11635 assert_eq!(
11636 verify_journalled_slot(&slot, nz(1), None)
11637 .unwrap_err()
11638 .refusal,
11639 SlotRefusal::Containment
11640 );
11641 assert!(
11642 sentinel.exists(),
11643 "the refusal resolved the junction and reached the operator's data"
11644 );
11645 assert!(
11646 slot.symlink_metadata().is_ok(),
11647 "the junction is left for the operator rather than removed as if it were ours"
11648 );
11649 }
11650
11651 #[cfg(windows)]
11660 fn plant_junction(link: &Path, target: &Path) -> Option<()> {
11661 let made = std::process::Command::new("cmd")
11662 .arg("/C")
11663 .arg("mklink")
11664 .arg("/J")
11665 .arg(link)
11666 .arg(target)
11667 .output()
11668 .ok()?;
11669 (made.status.success() && link.symlink_metadata().is_ok()).then_some(())
11670 }
11671
11672 #[test]
11677 fn removing_an_attempt_never_reaches_into_the_dependency_caches_it_linked() {
11678 let root = tempfile::tempdir().unwrap();
11679 let cache = root.path().join("_cache").join("o").join("r").join("npm");
11680 fs::create_dir_all(&cache).unwrap();
11681 let cached = cache.join("package.tgz");
11682 fs::write(&cached, b"kept between jobs").unwrap();
11683 let attempt = root.path().join("0123456789abcdef");
11684 let work = attempt.join("_work").join("repo");
11685 fs::create_dir_all(&work).unwrap();
11686 fs::hard_link(&cached, work.join("hard-linked.tgz")).unwrap();
11687 #[cfg(unix)]
11688 std::os::unix::fs::symlink(&cache, work.join("linked-cache")).unwrap();
11689 #[cfg(windows)]
11690 if plant_junction(&work.join("linked-cache"), &cache).is_none() {
11691 eprintln!("this machine would not create a junction; the hard link is still checked");
11692 }
11693
11694 remove_runtime_tree(&attempt).unwrap();
11695 assert!(!attempt.exists());
11696 assert_eq!(fs::read(&cached).unwrap(), b"kept between jobs");
11697 }
11698
11699 #[cfg(windows)]
11700 #[test]
11701 fn a_work_directory_replaced_by_a_junction_fails_closed_and_deletes_nothing_beyond_it() {
11702 let root = tempfile::tempdir().unwrap();
11703 let outside = root.path().join("operator-data");
11704 fs::create_dir(&outside).unwrap();
11705 let sentinel = outside.join("do-not-delete.txt");
11706 fs::write(
11707 &sentinel,
11708 b"an operator's data, outside every approved root",
11709 )
11710 .unwrap();
11711
11712 let slot = root.path().join("s1");
11713 fs::create_dir(&slot).unwrap();
11714 fs::create_dir(slot.join("bin")).unwrap();
11715 let Some(()) = plant_junction(&slot.join(DEFAULT_WORK_FOLDER), &outside) else {
11716 eprintln!("skipped: this machine would not create a directory junction");
11717 return;
11718 };
11719
11720 let work = fs::symlink_metadata(slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11724 assert!(is_link_like(&work), "a junction is a reparse point");
11725 let quarantine = scrub_slot_entries(&slot).unwrap_err();
11726 assert_eq!(quarantine.refusal, SlotRefusal::WorkNotADirectory);
11727 assert!(
11728 sentinel.exists(),
11729 "the deletion followed the junction out of the slot"
11730 );
11731 assert!(outside.is_dir());
11732
11733 let elsewhere = root.path().join("s2");
11736 fs::create_dir(&elsewhere).unwrap();
11737 fs::create_dir(elsewhere.join(DEFAULT_WORK_FOLDER)).unwrap();
11738 if plant_junction(&elsewhere.join("externals"), &outside).is_some() {
11739 scrub_slot_entries(&elsewhere).expect("an ordinary entry is removed, junction or not");
11740 verify_slot_scrubbed(&elsewhere).expect("and the slot verifies");
11741 assert!(sentinel.exists(), "the junction was followed, not unlinked");
11742 assert_eq!(entries_of(&elsewhere), only_the_job_workspace());
11743 }
11744 }
11745
11746 #[cfg(unix)]
11747 #[tokio::test]
11748 async fn a_substituted_work_directory_leaves_the_attempt_uncleaned_and_still_leased() {
11749 let harness = Harness::new(FakeGithubLifecycle::default(), Arc::new(PersistentDemand))
11750 .with_persistent_workspace(2);
11751 harness.ready().await;
11752 let first = harness.launch().await;
11753 let slot = harness.slot_path(1);
11754 harness.conclude(first.id);
11755
11756 let outside = harness._root.path().join("operator-data");
11757 fs::create_dir_all(&outside).unwrap();
11758 let sentinel = outside.join("do-not-delete.txt");
11759 fs::write(&sentinel, b"outside every approved root").unwrap();
11760 std::os::unix::fs::symlink(&outside, slot.join(DEFAULT_WORK_FOLDER)).unwrap();
11761
11762 harness
11763 .launcher
11764 .clean(first.id)
11765 .await
11766 .expect_err("a slot whose `_work` was substituted is quarantined");
11767 assert!(sentinel.exists());
11768
11769 let held = harness.attempt(first.id);
11770 assert_eq!(held.state(), AttemptState::Failed);
11771 assert!(held.holds_slot_lease());
11772
11773 let second = harness.launch().await;
11775 assert_eq!(second.workspace(), AttemptWorkspace::persistent_slot(nz(2)));
11776 }
11777
11778 #[test]
11779 fn a_slot_that_is_a_file_is_refused_and_a_slot_that_is_gone_is_not() {
11780 let root = tempfile::tempdir().unwrap();
11781 let occupied = root.path().join("s1");
11782 fs::write(&occupied, b"an operator's file").unwrap();
11783 verify_journalled_slot(&occupied, nz(1), None).expect("the path is the journalled slot");
11786 assert_eq!(
11787 slot_is_present(&occupied).unwrap_err().refusal,
11788 SlotRefusal::SlotNotADirectory
11789 );
11790 assert_eq!(fs::read_to_string(&occupied).unwrap(), "an operator's file");
11791
11792 assert!(!slot_is_present(&root.path().join("s2")).unwrap());
11795 let present = root.path().join("s3");
11796 fs::create_dir(&present).unwrap();
11797 assert!(slot_is_present(&present).unwrap());
11798 }
11799
11800 #[test]
11801 fn cleanup_dispatches_on_the_journalled_kind_and_not_on_what_the_directory_holds() {
11802 let root = tempfile::tempdir().unwrap();
11803
11804 let disposable = root.path().join("abcdef012345");
11808 fs::create_dir_all(disposable.join(DEFAULT_WORK_FOLDER).join("repo")).unwrap();
11809 let ephemeral = RunnerAttempt::allocate(
11810 AttemptId::new_random(),
11811 fixtures::POLICY_ID,
11812 &disposable,
11813 fixtures::created_at(),
11814 );
11815 remove_materialized_package(&ephemeral).unwrap();
11816 assert!(!disposable.exists());
11817
11818 let slot = root.path().join("s1");
11820 fs::create_dir_all(slot.join(DEFAULT_WORK_FOLDER).join("repo")).unwrap();
11821 fs::create_dir_all(slot.join("bin")).unwrap();
11822 let persistent = RunnerAttempt::allocate_in(
11823 AttemptId::new_random(),
11824 fixtures::POLICY_ID,
11825 &slot,
11826 AttemptWorkspace::persistent_slot(nz(1)),
11827 fixtures::created_at(),
11828 );
11829 remove_materialized_package(&persistent).unwrap();
11830 assert_eq!(entries_of(&slot), only_the_job_workspace());
11831 assert!(slot.join(DEFAULT_WORK_FOLDER).join("repo").is_dir());
11832 }
11833
11834 #[test]
11835 fn every_slot_refusal_names_a_distinct_event_class_and_keeps_the_lease() {
11836 let refusals = [
11837 SlotRefusal::NotTheJournalledSlot,
11838 SlotRefusal::PolicyRootDisagrees,
11839 SlotRefusal::Containment,
11840 SlotRefusal::SlotNotADirectory,
11841 SlotRefusal::Enumeration,
11842 SlotRefusal::WorkNotADirectory,
11843 SlotRefusal::Deletion,
11844 SlotRefusal::Residue,
11845 ];
11846 let classes: BTreeSet<&str> = refusals.iter().map(|refusal| refusal.class()).collect();
11847 assert_eq!(
11848 classes.len(),
11849 refusals.len(),
11850 "an event class shared by two refusals tells an operator less than it appears to"
11851 );
11852 for refusal in refusals {
11853 assert!(
11856 refusal
11857 .class()
11858 .chars()
11859 .all(|c| c.is_ascii_lowercase() || c == '_'),
11860 "{}",
11861 refusal.class()
11862 );
11863 assert!(
11864 refusal.remediation().contains("slot lease"),
11865 "every refusal has to say the lease is still held: {}",
11866 refusal.class()
11867 );
11868 }
11869 }
11870
11871 #[test]
11872 fn copy_package_tree_copies_files_and_preserves_paths_with_spaces() {
11873 let root = tempfile::tempdir().unwrap();
11874 let source = root.path().join("source with spaces");
11875 let dest = root.path().join("dest with spaces");
11876
11877 fs::create_dir_all(&source).unwrap();
11878 fs::write(source.join("file1.txt"), b"hello").unwrap();
11879
11880 let nested = source.join("nested dir");
11881 fs::create_dir_all(&nested).unwrap();
11882 fs::write(nested.join("file2.txt"), b"world").unwrap();
11883
11884 let nested_work = nested.join(DEFAULT_WORK_FOLDER);
11886 fs::create_dir_all(&nested_work).unwrap();
11887 fs::write(nested_work.join("allowed.txt"), b"allowed").unwrap();
11888
11889 copy_package_tree(&source, &dest, &TrustedTrees::default()).unwrap();
11890
11891 assert_eq!(fs::read_to_string(dest.join("file1.txt")).unwrap(), "hello");
11892 assert_eq!(
11893 fs::read_to_string(dest.join("nested dir").join("file2.txt")).unwrap(),
11894 "world"
11895 );
11896 assert_eq!(
11897 fs::read_to_string(
11898 dest.join("nested dir")
11899 .join(DEFAULT_WORK_FOLDER)
11900 .join("allowed.txt")
11901 )
11902 .unwrap(),
11903 "allowed"
11904 );
11905 }
11906
11907 #[test]
11908 fn every_field_name_the_attempt_sink_emits_is_allowed() {
11909 use runner_manager_platform::logging::is_field_allowed;
11910
11911 for field in [
11913 "event",
11914 "attempt_id",
11915 "reason",
11916 "retry_in_ms",
11917 "duration_ms",
11918 "order_wait_ms",
11919 "lock_wait_ms",
11920 "turn_wait_ms",
11921 "materialize_ms",
11922 "package_wait_ms",
11923 "package_install_ms",
11924 "package_verify_ms",
11925 "package_copy_ms",
11926 "package_downloaded",
11927 "prepare_ms",
11928 "register_ms",
11929 "spawn_ms",
11930 ] {
11931 assert!(
11932 is_field_allowed(field),
11933 "`{field}` would be logged as `[redacted]`"
11934 );
11935 }
11936 }
11937
11938 #[cfg(unix)]
11939 fn package_entry(root: &Path, version: &str, manifest: &str) -> PathBuf {
11940 let entry = root.join("packages").join(version);
11941 fs::create_dir_all(entry.join("bin")).unwrap();
11942 fs::write(entry.join(crate::package::MANIFEST_FILE), manifest).unwrap();
11943 fs::write(entry.join("bin").join("Runner.Listener"), version).unwrap();
11944 entry
11945 }
11946
11947 #[cfg(unix)]
11949 fn listener_inode(seed: &Path) -> u64 {
11950 use std::os::unix::fs::MetadataExt as _;
11951 fs::metadata(seed.join("bin").join("Runner.Listener"))
11952 .unwrap()
11953 .ino()
11954 }
11955
11956 #[cfg(unix)]
11957 #[test]
11958 fn a_package_seed_is_built_once_and_reused_while_untouched() {
11959 let root = tempfile::tempdir().unwrap();
11960 let source = package_entry(root.path(), "2.336.0", "{\"digest\":\"a\"}");
11961 let seeds = root.path().join("runners").join(PACKAGE_SEED_DIR);
11962 let trusted = TrustedTrees::default();
11963
11964 let seed = refresh_package_seed(&source, &seeds, &trusted).unwrap();
11965 assert_eq!(seed, seeds.join("2.336.0"));
11966 assert_eq!(
11967 fs::read_to_string(seed.join("bin").join("Runner.Listener")).unwrap(),
11968 "2.336.0"
11969 );
11970
11971 let built = listener_inode(&seed);
11972 assert_eq!(
11973 refresh_package_seed(&source, &seeds, &trusted).unwrap(),
11974 seed
11975 );
11976 assert_eq!(listener_inode(&seed), built, "an untouched seed is reused");
11977 }
11978
11979 #[cfg(unix)]
11980 #[test]
11981 fn a_tampered_or_unrecorded_seed_is_rebuilt() {
11982 let root = tempfile::tempdir().unwrap();
11983 let source = package_entry(root.path(), "2.336.0", "{\"digest\":\"a\"}");
11984 let seeds = root.path().join("runners").join(PACKAGE_SEED_DIR);
11985 let trusted = TrustedTrees::default();
11986 let seed = refresh_package_seed(&source, &seeds, &trusted).unwrap();
11987
11988 fs::write(seed.join("bin").join("Runner.Listener"), b"planted").unwrap();
11990 refresh_package_seed(&source, &seeds, &trusted).unwrap();
11991 assert_eq!(
11992 fs::read_to_string(seed.join("bin").join("Runner.Listener")).unwrap(),
11993 "2.336.0",
11994 "a seed whose print moved is rebuilt from the cache entry"
11995 );
11996
11997 let old_listener = fs::File::open(seed.join("bin").join("Runner.Listener")).unwrap();
12001 let built = listener_inode(&seed);
12002 refresh_package_seed(&source, &seeds, &TrustedTrees::default()).unwrap();
12003 assert_ne!(
12004 listener_inode(&seed),
12005 built,
12006 "an unrecorded seed is rebuilt"
12007 );
12008 drop(old_listener);
12009 }
12010
12011 #[cfg(unix)]
12012 #[test]
12013 fn a_package_seed_is_rebuilt_for_a_new_entry_and_drops_every_other() {
12014 let root = tempfile::tempdir().unwrap();
12015 let seeds = root.path().join("runners").join(PACKAGE_SEED_DIR);
12016 let trusted = TrustedTrees::default();
12017 let old = package_entry(root.path(), "2.335.0", "{\"digest\":\"old\"}");
12018 refresh_package_seed(&old, &seeds, &trusted).unwrap();
12019 fs::create_dir_all(seeds.join(".staging-interrupted")).unwrap();
12020 fs::write(seeds.join("stray"), b"").unwrap();
12021
12022 let new = package_entry(root.path(), "2.336.0", "{\"digest\":\"new\"}");
12023 let seed = refresh_package_seed(&new, &seeds, &trusted).unwrap();
12024
12025 let mut left: Vec<_> = fs::read_dir(&seeds)
12026 .unwrap()
12027 .map(|entry| entry.unwrap().file_name().into_string().unwrap())
12028 .collect();
12029 left.sort();
12030 assert_eq!(left, ["2.336.0"]);
12031
12032 fs::write(
12034 new.join(crate::package::MANIFEST_FILE),
12035 "{\"digest\":\"again\"}",
12036 )
12037 .unwrap();
12038 refresh_package_seed(&new, &seeds, &trusted).unwrap();
12039 assert_eq!(
12040 fs::read_to_string(seed.join(crate::package::MANIFEST_FILE)).unwrap(),
12041 "{\"digest\":\"again\"}",
12042 "a stale seed is rebuilt from the changed entry"
12043 );
12044 }
12045
12046 #[cfg(unix)]
12047 #[test]
12048 fn a_package_on_the_runtime_volume_is_its_own_seed() {
12049 let root = tempfile::tempdir().unwrap();
12050 let source = package_entry(root.path(), "2.336.0", "{}");
12051 let runtime = root.path().join("runners").join("attempt");
12052 fs::create_dir_all(&runtime).unwrap();
12053
12054 let trusted = TrustedTrees::default();
12055 assert_eq!(package_seed(&source, &runtime, &trusted).unwrap(), source);
12056 copy_package_tree(&source, &runtime, &trusted).unwrap();
12057 assert!(runtime.join("bin").join("Runner.Listener").exists());
12058 assert!(!root.path().join("runners").join(PACKAGE_SEED_DIR).exists());
12059 }
12060
12061 #[test]
12062 fn copy_package_tree_refuses_top_level_work_folder() {
12063 let root = tempfile::tempdir().unwrap();
12064 let source = root.path().join("source");
12065 let dest = root.path().join("dest");
12066
12067 fs::create_dir_all(&source).unwrap();
12068 fs::write(source.join("file1.txt"), b"hello").unwrap();
12069
12070 let top_work = source.join(DEFAULT_WORK_FOLDER);
12072 fs::create_dir_all(&top_work).unwrap();
12073
12074 let err = copy_package_tree(&source, &dest, &TrustedTrees::default()).unwrap_err();
12075 assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
12076 }
12077}