Skip to main content

sysprims_timeout/
managed.rs

1//! Native-owned managed containment lifecycle for language bindings.
2//!
3//! One registry owns every [`ContainmentGuard<Child>`] used by C/Go and N-API
4//! projections. Callers receive a generation-checked `u64` capability token,
5//! never a PID, Rust pointer, or reusable proof of spawn.
6
7use std::collections::BTreeMap;
8#[cfg(unix)]
9use std::process::{Command, Stdio};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock};
12use std::thread;
13use std::time::{Duration, Instant};
14
15use serde::Serialize;
16use sysprims_core::schema::CONTAINMENT_SNAPSHOT_V1;
17use sysprims_core::time::now_rfc3339;
18use sysprims_core::{get_platform, SysprimsError, SysprimsResult};
19
20use crate::{
21    ContainmentBoundaryStrength, ContainmentCompletionEvidence, ContainmentGuard,
22    ContainmentIdentity, ContainmentObservation, ContainmentOutcome, TerminateTreeConfig,
23    TreeKillReliability, SIGKILL, SIGTERM,
24};
25
26#[cfg(unix)]
27use crate::{spawn_contained, ContainmentSpawnError};
28
29pub const MAX_REGISTRY_SLOTS: usize = 1024;
30pub const MAX_ARGV_ENTRIES: usize = 256;
31pub const MAX_ARGV_ENTRY_BYTES: usize = 4096;
32pub const MAX_ARGV_TOTAL_BYTES: usize = 65_536;
33pub const MAX_ENV_ENTRIES: usize = 256;
34pub const MAX_ENV_KEY_BYTES: usize = 1024;
35pub const MAX_ENV_VALUE_BYTES: usize = 4096;
36pub const MAX_ENV_TOTAL_BYTES: usize = 65_536;
37pub const MAX_CWD_BYTES: usize = 4096;
38pub const MAX_DURATION_MS: u64 = 86_400_000;
39const POLL_INTERVAL: Duration = Duration::from_millis(10);
40
41#[derive(Debug, Clone)]
42pub struct ManagedSpawnRequest {
43    pub argv: Vec<String>,
44    pub cwd: Option<String>,
45    pub env: Option<BTreeMap<String, String>>,
46    pub execution_timeout_ms: Option<u64>,
47    pub grace_timeout_ms: Option<u64>,
48    pub kill_timeout_ms: Option<u64>,
49    pub signal: Option<i32>,
50    pub kill_signal: Option<i32>,
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54enum SlotState {
55    Empty,
56    Active,
57    Finalizing,
58    Inert,
59}
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62enum LeaderStatus {
63    Running,
64    Completed,
65    TimedOut,
66    Terminated,
67}
68
69impl LeaderStatus {
70    fn as_str(self) -> &'static str {
71        match self {
72            Self::Running => "running",
73            Self::Completed => "completed",
74            Self::TimedOut => "timed_out",
75            Self::Terminated => "terminated",
76        }
77    }
78}
79
80struct SlotData {
81    generation: u32,
82    retired: bool,
83    state: SlotState,
84    guard: Option<ContainmentGuard<ChildAdapter>>,
85    identity: Option<ContainmentIdentity>,
86    reliability: Option<TreeKillReliability>,
87    boundary: Option<ContainmentBoundaryStrength>,
88    terminate_config: TerminateTreeConfig,
89    outcome: Option<ContainmentOutcome>,
90    // Also records pending deadline enforcement while cleanup is retryable.
91    // Active snapshots still report Running until reap is confirmed.
92    leader_status: LeaderStatus,
93    deadline_cancel: Arc<AtomicBool>,
94    deadline_at: Option<Instant>,
95}
96
97struct Slot {
98    data: Mutex<SlotData>,
99    cond: Condvar,
100}
101
102struct Registry {
103    slots: Vec<Slot>,
104    free: Mutex<Vec<u32>>,
105}
106
107type ChildAdapter = std::process::Child;
108
109fn registry() -> &'static Registry {
110    static REGISTRY: OnceLock<Registry> = OnceLock::new();
111    REGISTRY.get_or_init(|| {
112        let mut slots = Vec::with_capacity(MAX_REGISTRY_SLOTS);
113        let mut free = Vec::with_capacity(MAX_REGISTRY_SLOTS);
114        for index in 0..MAX_REGISTRY_SLOTS {
115            slots.push(Slot {
116                data: Mutex::new(SlotData {
117                    generation: 1,
118                    retired: false,
119                    state: SlotState::Empty,
120                    guard: None,
121                    identity: None,
122                    reliability: None,
123                    boundary: None,
124                    terminate_config: TerminateTreeConfig::default(),
125                    outcome: None,
126                    leader_status: LeaderStatus::Running,
127                    deadline_cancel: Arc::new(AtomicBool::new(false)),
128                    deadline_at: None,
129                }),
130                cond: Condvar::new(),
131            });
132            free.push(index as u32);
133        }
134        Registry {
135            slots,
136            free: Mutex::new(free),
137        }
138    })
139}
140
141fn lock_poisoned() -> SysprimsError {
142    SysprimsError::internal("containment registry lock poisoned")
143}
144
145fn lock_data(slot: &Slot) -> SysprimsResult<MutexGuard<'_, SlotData>> {
146    slot.data.lock().map_err(|_| lock_poisoned())
147}
148
149fn lock_free(reg: &Registry) -> SysprimsResult<MutexGuard<'_, Vec<u32>>> {
150    reg.free.lock().map_err(|_| lock_poisoned())
151}
152
153#[cfg(unix)]
154fn pack_token(slot: u32, generation: u32) -> u64 {
155    ((generation as u64) << 32) | (u64::from(slot) + 1)
156}
157
158fn unpack_token(token: u64) -> SysprimsResult<(u32, u32)> {
159    if token == 0 {
160        return Err(invalid_handle("containment handle is invalid"));
161    }
162    let slot_plus_one = token & 0xffff_ffff;
163    if slot_plus_one == 0 {
164        return Err(invalid_handle("containment handle is invalid"));
165    }
166    let slot = slot_plus_one - 1;
167    if slot >= MAX_REGISTRY_SLOTS as u64 {
168        return Err(invalid_handle("containment handle is invalid"));
169    }
170    let generation = (token >> 32) as u32;
171    if generation == 0 {
172        return Err(invalid_handle("containment handle is invalid"));
173    }
174    Ok((slot as u32, generation))
175}
176
177fn invalid_handle(message: &str) -> SysprimsError {
178    SysprimsError::invalid_argument(message)
179}
180
181fn stale_handle() -> SysprimsError {
182    invalid_handle("containment handle is stale or closed")
183}
184
185#[cfg(unix)]
186fn spawn_error(error: ContainmentSpawnError) -> SysprimsError {
187    match error {
188        ContainmentSpawnError::Spawn(error) | ContainmentSpawnError::Adoption(error) => error,
189    }
190}
191
192pub fn validate_spawn_request(
193    request: &ManagedSpawnRequest,
194) -> SysprimsResult<TerminateTreeConfig> {
195    if request.argv.is_empty() {
196        return Err(SysprimsError::invalid_argument("argv must not be empty"));
197    }
198    if request.argv.len() > MAX_ARGV_ENTRIES {
199        return Err(SysprimsError::invalid_argument(format!(
200            "argv exceeds maximum of {MAX_ARGV_ENTRIES} entries"
201        )));
202    }
203    let mut argv_bytes = 0usize;
204    for (index, arg) in request.argv.iter().enumerate() {
205        if arg.is_empty() {
206            return Err(SysprimsError::invalid_argument(format!(
207                "argv[{index}] must not be empty"
208            )));
209        }
210        if arg.len() > MAX_ARGV_ENTRY_BYTES {
211            return Err(SysprimsError::invalid_argument(format!(
212                "argv[{index}] exceeds maximum of {MAX_ARGV_ENTRY_BYTES} bytes"
213            )));
214        }
215        argv_bytes = argv_bytes.saturating_add(arg.len());
216        if argv_bytes > MAX_ARGV_TOTAL_BYTES {
217            return Err(SysprimsError::invalid_argument(format!(
218                "argv exceeds maximum of {MAX_ARGV_TOTAL_BYTES} bytes"
219            )));
220        }
221    }
222
223    if let Some(cwd) = request.cwd.as_deref() {
224        if cwd.is_empty() {
225            return Err(SysprimsError::invalid_argument("cwd must not be empty"));
226        }
227        if cwd.len() > MAX_CWD_BYTES {
228            return Err(SysprimsError::invalid_argument(format!(
229                "cwd exceeds maximum of {MAX_CWD_BYTES} bytes"
230            )));
231        }
232    }
233
234    if let Some(env) = request.env.as_ref() {
235        if env.len() > MAX_ENV_ENTRIES {
236            return Err(SysprimsError::invalid_argument(format!(
237                "env exceeds maximum of {MAX_ENV_ENTRIES} entries"
238            )));
239        }
240        let mut env_bytes = 0usize;
241        for (key, value) in env {
242            if key.is_empty() {
243                return Err(SysprimsError::invalid_argument(
244                    "env keys must not be empty",
245                ));
246            }
247            if key.len() > MAX_ENV_KEY_BYTES {
248                return Err(SysprimsError::invalid_argument(format!(
249                    "env key exceeds maximum of {MAX_ENV_KEY_BYTES} bytes"
250                )));
251            }
252            if value.len() > MAX_ENV_VALUE_BYTES {
253                return Err(SysprimsError::invalid_argument(format!(
254                    "env value exceeds maximum of {MAX_ENV_VALUE_BYTES} bytes"
255                )));
256            }
257            env_bytes = env_bytes
258                .saturating_add(key.len())
259                .saturating_add(value.len());
260            if env_bytes > MAX_ENV_TOTAL_BYTES {
261                return Err(SysprimsError::invalid_argument(format!(
262                    "env exceeds maximum of {MAX_ENV_TOTAL_BYTES} bytes"
263                )));
264            }
265        }
266    }
267
268    if let Some(timeout) = request.execution_timeout_ms {
269        if timeout == 0 || timeout > MAX_DURATION_MS {
270            return Err(SysprimsError::invalid_argument(format!(
271                "execution_timeout_ms must be between 1 and {MAX_DURATION_MS}"
272            )));
273        }
274    }
275
276    let mut config = TerminateTreeConfig::default();
277    if let Some(grace) = request.grace_timeout_ms {
278        if grace > MAX_DURATION_MS {
279            return Err(SysprimsError::invalid_argument(format!(
280                "grace_timeout_ms exceeds maximum of {MAX_DURATION_MS}"
281            )));
282        }
283        config.grace_timeout_ms = grace;
284    }
285    if let Some(kill) = request.kill_timeout_ms {
286        if kill > MAX_DURATION_MS {
287            return Err(SysprimsError::invalid_argument(format!(
288                "kill_timeout_ms exceeds maximum of {MAX_DURATION_MS}"
289            )));
290        }
291        config.kill_timeout_ms = kill;
292    }
293    if let Some(signal) = request.signal {
294        validate_platform_signal(signal, "signal")?;
295        config.signal = signal;
296    }
297    if let Some(kill_signal) = request.kill_signal {
298        validate_platform_signal(kill_signal, "kill_signal")?;
299        config.kill_signal = kill_signal;
300    }
301    if config.signal == 0 {
302        config.signal = SIGTERM;
303    }
304    if config.kill_signal == 0 {
305        config.kill_signal = SIGKILL;
306    }
307    Ok(config)
308}
309
310pub(crate) fn validate_terminate_tree_config(config: &TerminateTreeConfig) -> SysprimsResult<()> {
311    if config.grace_timeout_ms > MAX_DURATION_MS {
312        return Err(SysprimsError::invalid_argument(format!(
313            "grace_timeout_ms exceeds maximum of {MAX_DURATION_MS}"
314        )));
315    }
316    if config.kill_timeout_ms > MAX_DURATION_MS {
317        return Err(SysprimsError::invalid_argument(format!(
318            "kill_timeout_ms exceeds maximum of {MAX_DURATION_MS}"
319        )));
320    }
321    validate_platform_signal(config.signal, "signal")?;
322    validate_platform_signal(config.kill_signal, "kill_signal")?;
323    Ok(())
324}
325
326fn platform_max_signal() -> i32 {
327    #[cfg(target_os = "linux")]
328    {
329        libc::SIGRTMAX()
330    }
331    #[cfg(not(target_os = "linux"))]
332    {
333        31
334    }
335}
336
337fn validate_platform_signal(signal: i32, name: &str) -> SysprimsResult<()> {
338    let max_signal = platform_max_signal();
339    if signal <= 0 || signal > max_signal {
340        return Err(SysprimsError::invalid_argument(format!(
341            "{name} must be between 1 and {max_signal}"
342        )));
343    }
344    Ok(())
345}
346
347fn cleanup_not_confirmed() -> SysprimsError {
348    SysprimsError::invalid_argument("containment cleanup did not confirm child reaped; retry")
349}
350
351#[cfg(unix)]
352fn allocate_slot() -> SysprimsResult<(u32, u32)> {
353    let reg = registry();
354    let mut free = lock_free(reg)?;
355    while let Some(slot_index) = free.pop() {
356        let slot = &reg.slots[slot_index as usize];
357        let data = lock_data(slot)?;
358        if data.retired || data.state != SlotState::Empty {
359            debug_assert!(
360                data.retired && data.state == SlotState::Empty,
361                "free list contained a live or inconsistent slot"
362            );
363            continue;
364        }
365        let generation = data.generation;
366        drop(data);
367        return Ok((slot_index, generation));
368    }
369    Err(SysprimsError::invalid_argument(
370        "containment registry has no free slots",
371    ))
372}
373
374/// Reset a slot to Empty and bump generation. Returns whether the index should
375/// be pushed onto the free list. Caller must drop the slot data lock before
376/// taking the free-list lock.
377fn reset_slot_for_reuse(data: &mut SlotData) -> bool {
378    data.guard = None;
379    data.identity = None;
380    data.reliability = None;
381    data.boundary = None;
382    data.outcome = None;
383    data.leader_status = LeaderStatus::Running;
384    data.deadline_cancel.store(true, Ordering::SeqCst);
385    data.deadline_cancel = Arc::new(AtomicBool::new(false));
386    data.deadline_at = None;
387    data.state = SlotState::Empty;
388    if data.generation == u32::MAX {
389        data.retired = true;
390        return false;
391    }
392    let next = data.generation.saturating_add(1);
393    if next == 0 || next == u32::MAX {
394        data.generation = u32::MAX;
395        data.retired = true;
396        return false;
397    }
398    data.generation = next;
399    true
400}
401
402fn push_free_index(slot_index: u32) -> SysprimsResult<()> {
403    let mut free = lock_free(registry())?;
404    free.push(slot_index);
405    Ok(())
406}
407
408fn recycle_after_data_unlock(
409    slot: &Slot,
410    slot_index: u32,
411    mut data: MutexGuard<'_, SlotData>,
412) -> SysprimsResult<()> {
413    let should_push = reset_slot_for_reuse(&mut data);
414    slot.cond.notify_all();
415    drop(data);
416    if should_push {
417        push_free_index(slot_index)?;
418    }
419    Ok(())
420}
421
422fn lookup<'a>(token: u64) -> SysprimsResult<(u32, u32, &'a Slot)> {
423    let (slot_index, generation) = unpack_token(token)?;
424    let slot = &registry().slots[slot_index as usize];
425    Ok((slot_index, generation, slot))
426}
427
428fn snapshot_from(
429    data: &SlotData,
430    handle_state: &'static str,
431    leader_status: LeaderStatus,
432) -> SysprimsResult<ManagedSnapshot> {
433    let identity = data
434        .identity
435        .as_ref()
436        .ok_or_else(|| SysprimsError::internal("containment identity missing"))?;
437    let reliability = data
438        .reliability
439        .ok_or_else(|| SysprimsError::internal("containment reliability missing"))?;
440    let boundary = data
441        .boundary
442        .ok_or_else(|| SysprimsError::internal("containment boundary missing"))?;
443    let mut snapshot = ManagedSnapshot {
444        schema_id: CONTAINMENT_SNAPSHOT_V1,
445        timestamp: now_rfc3339(),
446        platform: get_platform(),
447        handle_state,
448        leader_status: leader_status.as_str(),
449        identity: ManagedIdentity {
450            pid: identity.pid,
451            start_time_unix_ms: identity.start_time_unix_ms,
452            exe_path: identity.exe_path.clone(),
453        },
454        tree_kill_reliability: reliability.as_str(),
455        boundary_strength: boundary.as_str(),
456        pgid: None,
457        signal_sent: None,
458        kill_signal: None,
459        escalated: None,
460        exited: None,
461        timed_out: None,
462        completion: None,
463        warnings: Vec::new(),
464    };
465    if let Some(outcome) = data.outcome.as_ref() {
466        snapshot.pgid = outcome.pgid;
467        snapshot.signal_sent = outcome.signal_sent;
468        snapshot.kill_signal = outcome.kill_signal;
469        snapshot.escalated = Some(outcome.escalated);
470        snapshot.exited = Some(outcome.exited);
471        snapshot.timed_out = Some(outcome.timed_out);
472        snapshot.completion = Some(outcome.completion.clone());
473        snapshot.warnings = outcome.warnings.clone();
474    }
475    Ok(snapshot)
476}
477
478fn commit_inert(
479    slot: &Slot,
480    data: &mut SlotData,
481    outcome: ContainmentOutcome,
482    leader_status: LeaderStatus,
483) -> SysprimsResult<ManagedSnapshot> {
484    data.deadline_cancel.store(true, Ordering::SeqCst);
485    data.outcome = Some(outcome);
486    data.leader_status = leader_status;
487    data.state = SlotState::Inert;
488    slot.cond.notify_all();
489    snapshot_from(data, "inert", leader_status)
490}
491
492/// Spawn a sysprims-owned contained child and return a capability token.
493pub fn spawn(request: ManagedSpawnRequest) -> SysprimsResult<u64> {
494    let terminate_config = validate_spawn_request(&request)?;
495
496    #[cfg(windows)]
497    {
498        let _ = terminate_config;
499        return Err(SysprimsError::not_supported(
500            "managed containment spawn without create-suspended Job assignment",
501            "windows",
502        ));
503    }
504
505    #[cfg(unix)]
506    {
507        let (slot_index, generation) = allocate_slot()?;
508        let token = pack_token(slot_index, generation);
509        let pending_deadline = match request.execution_timeout_ms {
510            Some(timeout_ms) => match start_unarmed_deadline_monitor() {
511                Ok(monitor) => Some((timeout_ms, monitor)),
512                Err(error) => {
513                    let _ = recycle_unused_slot(slot_index, generation);
514                    return Err(error);
515                }
516            },
517            None => None,
518        };
519
520        let command = match build_command(&request) {
521            Ok(command) => command,
522            Err(error) => {
523                abandon_unused_construction(slot_index, generation, pending_deadline.as_ref());
524                return Err(error);
525            }
526        };
527
528        let guard = match spawn_contained(command) {
529            Ok(guard) => guard,
530            Err(error) => {
531                abandon_unused_construction(slot_index, generation, pending_deadline.as_ref());
532                return Err(spawn_error(error));
533            }
534        };
535
536        let reliability = guard.tree_kill_reliability();
537        let boundary = guard.boundary_strength();
538        if reliability != TreeKillReliability::Guaranteed
539            || boundary != ContainmentBoundaryStrength::CooperativeGroup
540        {
541            drop(guard);
542            abandon_unused_construction(slot_index, generation, pending_deadline.as_ref());
543            return Err(SysprimsError::internal(
544                "managed spawn produced a non-guaranteed cooperative-group guard",
545            ));
546        }
547
548        let identity = guard.identity().clone();
549        let slot = &registry().slots[slot_index as usize];
550        let deadline_at = pending_deadline
551            .as_ref()
552            .map(|(timeout_ms, _)| Instant::now() + Duration::from_millis(*timeout_ms));
553        {
554            let mut data = lock_data(slot)?;
555            if data.generation != generation || data.state != SlotState::Empty {
556                drop(guard);
557                if let Some((_, monitor)) = pending_deadline.as_ref() {
558                    monitor.cancel();
559                }
560                return Err(SysprimsError::internal(
561                    "containment slot changed before spawn insert",
562                ));
563            }
564            data.identity = Some(identity);
565            data.reliability = Some(reliability);
566            data.boundary = Some(boundary);
567            data.terminate_config = terminate_config;
568            data.leader_status = LeaderStatus::Running;
569            data.outcome = None;
570            if let Some((_, monitor)) = pending_deadline.as_ref() {
571                data.deadline_cancel = Arc::clone(&monitor.cancel);
572            }
573            data.deadline_at = deadline_at;
574            data.guard = Some(guard);
575            data.state = SlotState::Active;
576        }
577        if let Some((_, monitor)) = pending_deadline {
578            if let Some(deadline) = deadline_at {
579                monitor.arm(token, deadline);
580            }
581        }
582        Ok(token)
583    }
584}
585
586#[cfg(unix)]
587fn abandon_unused_construction(
588    slot_index: u32,
589    generation: u32,
590    pending_deadline: Option<&(u64, DeadlineMonitor)>,
591) {
592    if let Some((_, monitor)) = pending_deadline {
593        monitor.cancel();
594    }
595    let _ = recycle_unused_slot(slot_index, generation);
596}
597
598#[cfg(unix)]
599fn recycle_unused_slot(slot_index: u32, generation: u32) -> SysprimsResult<()> {
600    let reg = registry();
601    let slot = &reg.slots[slot_index as usize];
602    let data = lock_data(slot)?;
603    if data.generation == generation && data.state == SlotState::Empty && !data.retired {
604        drop(data);
605        push_free_index(slot_index)?;
606    }
607    Ok(())
608}
609
610#[cfg(unix)]
611fn build_command(request: &ManagedSpawnRequest) -> SysprimsResult<Command> {
612    let mut command = Command::new(&request.argv[0]);
613    if request.argv.len() > 1 {
614        command.args(&request.argv[1..]);
615    }
616    command
617        .stdin(Stdio::null())
618        .stdout(Stdio::null())
619        .stderr(Stdio::null());
620    if let Some(cwd) = request.cwd.as_deref() {
621        command.current_dir(cwd);
622    }
623    if let Some(env) = request.env.as_ref() {
624        for (key, value) in env {
625            command.env(key, value);
626        }
627    }
628    Ok(command)
629}
630
631#[derive(Clone, Copy)]
632struct DeadlineArm {
633    token: u64,
634    deadline: Instant,
635}
636
637#[cfg(unix)]
638struct DeadlineMonitor {
639    arm: Arc<Mutex<Option<DeadlineArm>>>,
640    cancel: Arc<AtomicBool>,
641}
642
643#[cfg(unix)]
644impl DeadlineMonitor {
645    fn cancel(&self) {
646        self.cancel.store(true, Ordering::SeqCst);
647    }
648
649    /// Arm is infallible: the monitor thread is already reserved, and this
650    /// mutex has no panicking critical section. Recover poison and own the
651    /// arm state so a returned handle always has a deadline.
652    fn arm(&self, token: u64, deadline: Instant) {
653        *lock_arm(&self.arm) = Some(DeadlineArm { token, deadline });
654    }
655}
656
657fn lock_arm(arm: &Mutex<Option<DeadlineArm>>) -> MutexGuard<'_, Option<DeadlineArm>> {
658    arm.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
659}
660
661#[cfg(unix)]
662fn start_unarmed_deadline_monitor() -> SysprimsResult<DeadlineMonitor> {
663    if test_force_deadline_thread_fail() {
664        return Err(SysprimsError::system(
665            "failed to start containment deadline monitor",
666            0,
667        ));
668    }
669    let arm = Arc::new(Mutex::new(None));
670    let cancel = Arc::new(AtomicBool::new(false));
671    spawn_deadline_worker(Arc::clone(&arm), Arc::clone(&cancel))?;
672    Ok(DeadlineMonitor { arm, cancel })
673}
674
675fn spawn_deadline_worker(
676    arm: Arc<Mutex<Option<DeadlineArm>>>,
677    cancel: Arc<AtomicBool>,
678) -> SysprimsResult<()> {
679    thread::Builder::new()
680        .name("sysprims-containment-deadline".into())
681        .spawn(move || run_deadline_monitor(arm, cancel))
682        .map_err(|error| {
683            SysprimsError::system(
684                format!("failed to start containment deadline monitor: {error}"),
685                error.raw_os_error().unwrap_or(0),
686            )
687        })?;
688    Ok(())
689}
690
691fn run_deadline_monitor(arm: Arc<Mutex<Option<DeadlineArm>>>, cancel: Arc<AtomicBool>) {
692    loop {
693        if cancel.load(Ordering::SeqCst) {
694            return;
695        }
696        let armed = *lock_arm(&arm);
697        match armed {
698            Some(DeadlineArm { token, deadline }) => {
699                let now = Instant::now();
700                if now >= deadline {
701                    if terminate_with_status(token, LeaderStatus::TimedOut).is_ok() {
702                        return;
703                    }
704                    // Errors and unconfirmed reap retain the active capability.
705                    // Keep its existing monitor alive until finalization or close
706                    // cancels it; retry against the same generation-bound token.
707                    thread::sleep(POLL_INTERVAL);
708                    continue;
709                }
710                thread::sleep(POLL_INTERVAL.min(deadline.saturating_duration_since(now)));
711            }
712            None => thread::sleep(POLL_INTERVAL),
713        }
714    }
715}
716
717fn restore_deadline_monitor(data: &mut SlotData, token: u64) {
718    let Some(deadline) = data.deadline_at else {
719        return;
720    };
721    if !data.deadline_cancel.load(Ordering::SeqCst) {
722        return;
723    }
724    let cancel = Arc::new(AtomicBool::new(false));
725    let arm = Arc::new(Mutex::new(Some(DeadlineArm { token, deadline })));
726    if spawn_deadline_worker(Arc::clone(&arm), Arc::clone(&cancel)).is_ok() {
727        data.deadline_cancel = cancel;
728    }
729}
730
731#[cfg(unix)]
732fn test_force_deadline_thread_fail() -> bool {
733    #[cfg(test)]
734    {
735        inject::FORCE_DEADLINE_THREAD_FAIL.swap(false, Ordering::SeqCst)
736    }
737    #[cfg(not(test))]
738    {
739        false
740    }
741}
742
743fn test_force_unreaped() -> bool {
744    #[cfg(test)]
745    {
746        inject::FORCE_UNREAPED.swap(false, Ordering::SeqCst)
747    }
748    #[cfg(not(test))]
749    {
750        false
751    }
752}
753
754fn test_force_operation_error() -> bool {
755    #[cfg(test)]
756    {
757        inject::FORCE_OPERATION_ERROR.swap(false, Ordering::SeqCst)
758    }
759    #[cfg(not(test))]
760    {
761        false
762    }
763}
764
765#[cfg(test)]
766mod inject {
767    use std::sync::atomic::AtomicBool;
768    use std::sync::Mutex;
769
770    pub static FORCE_UNREAPED: AtomicBool = AtomicBool::new(false);
771    pub static FORCE_OPERATION_ERROR: AtomicBool = AtomicBool::new(false);
772    pub static FORCE_DEADLINE_THREAD_FAIL: AtomicBool = AtomicBool::new(false);
773    pub static TEST_MUTEX: Mutex<()> = Mutex::new(());
774}
775
776fn unreaped_outcome(data: &SlotData) -> SysprimsResult<ContainmentOutcome> {
777    let identity = data
778        .identity
779        .clone()
780        .ok_or_else(|| SysprimsError::internal("containment identity missing"))?;
781    let reliability = data
782        .reliability
783        .ok_or_else(|| SysprimsError::internal("containment reliability missing"))?;
784    let boundary = data
785        .boundary
786        .ok_or_else(|| SysprimsError::internal("containment boundary missing"))?;
787    Ok(ContainmentOutcome {
788        identity,
789        pgid: None,
790        signal_sent: None,
791        kill_signal: None,
792        escalated: false,
793        exited: false,
794        timed_out: true,
795        tree_kill_reliability: reliability,
796        boundary_strength: boundary,
797        completion: ContainmentCompletionEvidence::Unknown {
798            observation: ContainmentObservation::UnsupportedPlatform,
799        },
800        warnings: vec!["Timed out waiting to reap contained child".to_string()],
801    })
802}
803
804fn wait_for_not_finalizing<'a>(
805    slot: &'a Slot,
806    mut data: MutexGuard<'a, SlotData>,
807    wait_deadline: Option<Instant>,
808) -> SysprimsResult<MutexGuard<'a, SlotData>> {
809    let generation = data.generation;
810    while data.state == SlotState::Finalizing && data.generation == generation {
811        if let Some(deadline) = wait_deadline {
812            let now = Instant::now();
813            if now >= deadline {
814                return Ok(data);
815            }
816            let remaining = deadline.saturating_duration_since(now);
817            let (guard, _) = slot
818                .cond
819                .wait_timeout(data, remaining)
820                .map_err(|_| lock_poisoned())?;
821            data = guard;
822        } else {
823            data = slot.cond.wait(data).map_err(|_| lock_poisoned())?;
824        }
825    }
826    Ok(data)
827}
828
829pub fn identity(token: u64) -> SysprimsResult<ManagedSnapshot> {
830    let (_slot_index, generation, slot) = lookup(token)?;
831    let data = lock_data(slot)?;
832    if data.generation != generation || data.retired {
833        return Err(stale_handle());
834    }
835    match data.state {
836        SlotState::Empty => Err(stale_handle()),
837        SlotState::Active | SlotState::Finalizing => {
838            snapshot_from(&data, "active", LeaderStatus::Running)
839        }
840        SlotState::Inert => snapshot_from(&data, "inert", data.leader_status),
841    }
842}
843
844pub fn poll(token: u64) -> SysprimsResult<ManagedSnapshot> {
845    operate_non_blocking(
846        token,
847        LeaderStatus::Completed,
848        Some(Instant::now()),
849        |guard, config| guard.try_complete(config),
850    )
851}
852
853pub fn wait(token: u64, wait_timeout_ms: u64) -> SysprimsResult<ManagedSnapshot> {
854    if wait_timeout_ms > MAX_DURATION_MS {
855        return Err(SysprimsError::invalid_argument(format!(
856            "wait timeout exceeds maximum of {MAX_DURATION_MS}"
857        )));
858    }
859    let wait_deadline = if wait_timeout_ms == 0 {
860        None
861    } else {
862        Some(Instant::now() + Duration::from_millis(wait_timeout_ms))
863    };
864
865    loop {
866        match operate_non_blocking(
867            token,
868            LeaderStatus::Completed,
869            wait_deadline,
870            |guard, config| guard.try_complete(config),
871        ) {
872            Ok(snapshot) if snapshot.leader_status != "running" => return Ok(snapshot),
873            Ok(snapshot) => {
874                if let Some(deadline) = wait_deadline {
875                    if Instant::now() >= deadline {
876                        return Ok(snapshot);
877                    }
878                }
879                let remaining = wait_deadline
880                    .map(|deadline| deadline.saturating_duration_since(Instant::now()));
881                thread::sleep(
882                    remaining.map_or(POLL_INTERVAL, |remaining| POLL_INTERVAL.min(remaining)),
883                );
884            }
885            Err(error) => return Err(error),
886        }
887    }
888}
889
890pub fn terminate(token: u64) -> SysprimsResult<ManagedSnapshot> {
891    terminate_with_status(token, LeaderStatus::Terminated)
892}
893
894fn terminate_with_status(
895    token: u64,
896    leader_status: LeaderStatus,
897) -> SysprimsResult<ManagedSnapshot> {
898    operate_blocking(token, leader_status, |guard, config| {
899        guard.terminate(config).map(Some)
900    })
901}
902
903pub fn close(token: u64) -> SysprimsResult<()> {
904    let (slot_index, generation, slot) = lookup(token)?;
905    let mut data = lock_data(slot)?;
906    if data.generation != generation || data.retired {
907        return Err(stale_handle());
908    }
909    data = wait_for_not_finalizing(slot, data, Some(Instant::now() + Duration::from_secs(120)))?;
910    if data.generation != generation {
911        return Err(stale_handle());
912    }
913    match data.state {
914        SlotState::Empty => Err(stale_handle()),
915        SlotState::Finalizing => Err(SysprimsError::invalid_argument(
916            "containment handle is busy; retry close",
917        )),
918        SlotState::Inert => recycle_after_data_unlock(slot, slot_index, data),
919        SlotState::Active => {
920            let mut guard = data
921                .guard
922                .take()
923                .ok_or_else(|| SysprimsError::internal("containment guard missing during close"))?;
924            let config = data.terminate_config.clone();
925            let forced = test_force_unreaped().then(|| unreaped_outcome(&data));
926            data.state = SlotState::Finalizing;
927            drop(data);
928            // Finalizing reserves this generation; only this owner holds the
929            // guard. Release the state lock so bounded waiters can observe it.
930            let result = match forced {
931                Some(outcome) => outcome.map(Some),
932                None => guard.terminate(config).map(Some),
933            };
934            let mut data = lock_data(slot)?;
935            data.guard = Some(guard);
936            match result {
937                Ok(Some(outcome)) if outcome.exited => {
938                    recycle_after_data_unlock(slot, slot_index, data)
939                }
940                Ok(_) => {
941                    data.state = SlotState::Active;
942                    restore_deadline_monitor(&mut data, token);
943                    slot.cond.notify_all();
944                    Err(cleanup_not_confirmed())
945                }
946                Err(error) => {
947                    data.state = SlotState::Active;
948                    restore_deadline_monitor(&mut data, token);
949                    slot.cond.notify_all();
950                    Err(error)
951                }
952            }
953        }
954    }
955}
956
957fn operate_non_blocking(
958    token: u64,
959    complete_status: LeaderStatus,
960    wait_deadline: Option<Instant>,
961    op: impl FnOnce(
962        &mut ContainmentGuard<ChildAdapter>,
963        TerminateTreeConfig,
964    ) -> SysprimsResult<Option<ContainmentOutcome>>,
965) -> SysprimsResult<ManagedSnapshot> {
966    operate_inner(token, complete_status, wait_deadline, op)
967}
968
969fn operate_blocking(
970    token: u64,
971    complete_status: LeaderStatus,
972    op: impl FnOnce(
973        &mut ContainmentGuard<ChildAdapter>,
974        TerminateTreeConfig,
975    ) -> SysprimsResult<Option<ContainmentOutcome>>,
976) -> SysprimsResult<ManagedSnapshot> {
977    operate_inner(token, complete_status, None, op)
978}
979
980fn operate_inner(
981    token: u64,
982    mut complete_status: LeaderStatus,
983    wait_deadline: Option<Instant>,
984    op: impl FnOnce(
985        &mut ContainmentGuard<ChildAdapter>,
986        TerminateTreeConfig,
987    ) -> SysprimsResult<Option<ContainmentOutcome>>,
988) -> SysprimsResult<ManagedSnapshot> {
989    let (_slot_index, generation, slot) = lookup(token)?;
990    let mut data = lock_data(slot)?;
991    if data.generation != generation || data.retired {
992        return Err(stale_handle());
993    }
994    let waited_for_owner = data.state == SlotState::Finalizing;
995    data = wait_for_not_finalizing(slot, data, wait_deadline)?;
996    if data.generation != generation {
997        return Err(stale_handle());
998    }
999    // A failed owner may restore Active as this wait expires. Do not acquire
1000    // cleanup ownership after spending the caller's budget waiting for it.
1001    if waited_for_owner
1002        && data.state == SlotState::Active
1003        && wait_deadline.is_some_and(|deadline| Instant::now() >= deadline)
1004    {
1005        return snapshot_from(&data, "active", LeaderStatus::Running);
1006    }
1007    match data.state {
1008        SlotState::Empty => Err(stale_handle()),
1009        SlotState::Finalizing => snapshot_from(&data, "active", LeaderStatus::Running),
1010        SlotState::Inert => snapshot_from(&data, "inert", data.leader_status),
1011        SlotState::Active => {
1012            let mut guard = data.guard.take().ok_or_else(|| {
1013                SysprimsError::internal("containment guard missing during operation")
1014            })?;
1015            let config = data.terminate_config.clone();
1016            let forced = test_force_unreaped().then(|| unreaped_outcome(&data));
1017            let mut pending_status = data.leader_status;
1018            data.state = SlotState::Finalizing;
1019            drop(data);
1020            // This owner exclusively holds the guard until it restores the
1021            // slot. Other operations may observe/wait, but cannot recycle it.
1022            let result = (|| {
1023                if complete_status == LeaderStatus::TimedOut
1024                    && pending_status != LeaderStatus::TimedOut
1025                {
1026                    if let Some(outcome) = guard.try_complete(config.clone())? {
1027                        complete_status = LeaderStatus::Completed;
1028                        return Ok(Some(outcome));
1029                    }
1030                    pending_status = LeaderStatus::TimedOut;
1031                }
1032                if test_force_operation_error() {
1033                    return Err(SysprimsError::internal(
1034                        "injected containment operation failure",
1035                    ));
1036                }
1037                if let Some(outcome) = forced {
1038                    return outcome.map(Some);
1039                }
1040                op(&mut guard, config)
1041            })();
1042            let mut data = lock_data(slot)?;
1043            data.guard = Some(guard);
1044            data.leader_status = pending_status;
1045            match result {
1046                Ok(Some(outcome)) if outcome.exited => {
1047                    if data.leader_status == LeaderStatus::TimedOut {
1048                        complete_status = LeaderStatus::TimedOut;
1049                    }
1050                    commit_inert(slot, &mut data, outcome, complete_status)
1051                }
1052                Ok(Some(_outcome)) => {
1053                    data.state = SlotState::Active;
1054                    slot.cond.notify_all();
1055                    Err(cleanup_not_confirmed())
1056                }
1057                Ok(None) => {
1058                    data.state = SlotState::Active;
1059                    slot.cond.notify_all();
1060                    snapshot_from(&data, "active", LeaderStatus::Running)
1061                }
1062                Err(error) => {
1063                    data.state = SlotState::Active;
1064                    slot.cond.notify_all();
1065                    Err(error)
1066                }
1067            }
1068        }
1069    }
1070}
1071
1072#[derive(Debug, Clone, Serialize)]
1073pub struct ManagedIdentity {
1074    pub pid: u32,
1075    pub start_time_unix_ms: u64,
1076    pub exe_path: String,
1077}
1078
1079#[derive(Debug, Clone, Serialize)]
1080pub struct ManagedSnapshot {
1081    pub schema_id: &'static str,
1082    pub timestamp: String,
1083    pub platform: &'static str,
1084    pub handle_state: &'static str,
1085    pub leader_status: &'static str,
1086    pub identity: ManagedIdentity,
1087    pub tree_kill_reliability: &'static str,
1088    pub boundary_strength: &'static str,
1089    #[serde(skip_serializing_if = "Option::is_none")]
1090    pub pgid: Option<u32>,
1091    #[serde(skip_serializing_if = "Option::is_none")]
1092    pub signal_sent: Option<i32>,
1093    #[serde(skip_serializing_if = "Option::is_none")]
1094    pub kill_signal: Option<i32>,
1095    #[serde(skip_serializing_if = "Option::is_none")]
1096    pub escalated: Option<bool>,
1097    #[serde(skip_serializing_if = "Option::is_none")]
1098    pub exited: Option<bool>,
1099    #[serde(skip_serializing_if = "Option::is_none")]
1100    pub timed_out: Option<bool>,
1101    #[serde(skip_serializing_if = "Option::is_none")]
1102    pub completion: Option<ContainmentCompletionEvidence>,
1103    pub warnings: Vec<String>,
1104}
1105
1106#[cfg(test)]
1107mod tests {
1108    use super::*;
1109    use std::sync::atomic::Ordering;
1110    #[cfg(unix)]
1111    use std::sync::mpsc;
1112    use std::sync::MutexGuard;
1113    #[cfg(unix)]
1114    use std::thread;
1115
1116    fn test_guard() -> MutexGuard<'static, ()> {
1117        let guard = inject::TEST_MUTEX
1118            .lock()
1119            .unwrap_or_else(|poisoned| poisoned.into_inner());
1120        inject::FORCE_UNREAPED.store(false, Ordering::SeqCst);
1121        inject::FORCE_OPERATION_ERROR.store(false, Ordering::SeqCst);
1122        inject::FORCE_DEADLINE_THREAD_FAIL.store(false, Ordering::SeqCst);
1123        guard
1124    }
1125
1126    #[cfg(unix)]
1127    fn occupied_slots() -> usize {
1128        registry()
1129            .slots
1130            .iter()
1131            .filter(|slot| {
1132                lock_data(slot)
1133                    .map(|data| data.state != SlotState::Empty)
1134                    .unwrap_or(true)
1135            })
1136            .count()
1137    }
1138
1139    fn request(argv: Vec<&str>) -> ManagedSpawnRequest {
1140        ManagedSpawnRequest {
1141            argv: argv.into_iter().map(str::to_string).collect(),
1142            cwd: None,
1143            env: None,
1144            execution_timeout_ms: None,
1145            grace_timeout_ms: Some(50),
1146            kill_timeout_ms: Some(200),
1147            signal: None,
1148            kill_signal: None,
1149        }
1150    }
1151
1152    #[test]
1153    fn rejects_empty_argv_and_signal_zero() {
1154        let _lock = test_guard();
1155        assert!(validate_spawn_request(&request(vec![])).is_err());
1156        let mut bad = request(vec!["true"]);
1157        bad.signal = Some(0);
1158        assert!(validate_spawn_request(&bad).is_err());
1159        let mut overflow = request(vec!["true"]);
1160        overflow.execution_timeout_ms = Some(MAX_DURATION_MS + 1);
1161        assert!(validate_spawn_request(&overflow).is_err());
1162        let mut high = request(vec!["true"]);
1163        high.signal = Some(99);
1164        assert!(validate_spawn_request(&high).is_err());
1165        let mut high_kill = request(vec!["true"]);
1166        high_kill.kill_signal = Some(99);
1167        assert!(validate_spawn_request(&high_kill).is_err());
1168    }
1169
1170    #[test]
1171    fn unpack_rejects_zero_and_malformed_tokens() {
1172        let _lock = test_guard();
1173        assert!(unpack_token(0).is_err());
1174        assert!(unpack_token(1u64 << 32).is_err());
1175        assert!(identity(0).is_err());
1176        assert!(identity(1).is_err());
1177        assert!(identity(u64::MAX).is_err());
1178        assert!(close(0).is_err());
1179    }
1180
1181    #[cfg(unix)]
1182    #[test]
1183    fn spawn_wait_complete_dispose_twice_and_stale_reuse() {
1184        let _lock = test_guard();
1185        let token = spawn(request(vec!["true"])).expect("spawn true");
1186        let snap = wait(token, 5_000).expect("wait true");
1187        assert_eq!(snap.leader_status, "completed");
1188        assert_eq!(snap.tree_kill_reliability, "guaranteed");
1189        assert_eq!(snap.boundary_strength, "cooperative_group");
1190        assert_eq!(snap.handle_state, "inert");
1191        match snap.completion {
1192            Some(ContainmentCompletionEvidence::Empty { .. })
1193            | Some(ContainmentCompletionEvidence::Unknown { .. }) => {}
1194            other => panic!("unexpected completion: {other:?}"),
1195        }
1196        close(token).expect("close");
1197        assert!(
1198            close(token).is_err(),
1199            "closed token must fail closed at FFI"
1200        );
1201        let token2 = spawn(request(vec!["true"])).expect("reuse slot");
1202        assert_ne!(token, token2);
1203        assert!(
1204            identity(token).is_err(),
1205            "stale token must not follow slot reuse"
1206        );
1207        close(token2).ok();
1208    }
1209
1210    #[cfg(unix)]
1211    #[test]
1212    fn terminate_keeps_spawn_reliability_and_linearizes_waiters() {
1213        let _lock = test_guard();
1214        let token = spawn(request(vec!["sleep", "30"])).expect("spawn sleep");
1215        let identity_before = identity(token).expect("identity");
1216        let snap = terminate(token).expect("terminate");
1217        assert_eq!(snap.leader_status, "terminated");
1218        assert_eq!(
1219            snap.tree_kill_reliability,
1220            identity_before.tree_kill_reliability
1221        );
1222        let again = terminate(token).expect("inert terminate");
1223        assert_eq!(again.handle_state, "inert");
1224        assert_eq!(again.leader_status, "terminated");
1225        close(token).expect("close after terminate");
1226    }
1227
1228    #[cfg(unix)]
1229    #[test]
1230    fn native_deadline_times_out_without_language_wait() {
1231        let _lock = test_guard();
1232        let mut req = request(vec!["sleep", "30"]);
1233        req.execution_timeout_ms = Some(200);
1234        req.grace_timeout_ms = Some(20);
1235        let token = spawn(req).expect("spawn with deadline");
1236        let snap = wait(token, 5_000).expect("wait for native deadline");
1237        assert_eq!(snap.leader_status, "timed_out");
1238        assert_eq!(snap.tree_kill_reliability, "guaranteed");
1239        close(token).ok();
1240    }
1241
1242    #[cfg(unix)]
1243    #[test]
1244    fn bounded_wait_does_not_wait_for_finalizing_owner() {
1245        let _lock = test_guard();
1246        let token = spawn(request(vec!["sleep", "30"])).expect("spawn owned child");
1247        let (entered_tx, entered_rx) = mpsc::channel();
1248        let (release_tx, release_rx) = mpsc::channel();
1249        let owner = thread::spawn(move || {
1250            operate_blocking(token, LeaderStatus::Terminated, |guard, config| {
1251                entered_tx.send(()).unwrap();
1252                // Bound the fixture even if the waiter regresses and blocks.
1253                let _ = release_rx.recv_timeout(Duration::from_secs(1));
1254                guard.terminate(config).map(Some)
1255            })
1256        });
1257        entered_rx.recv_timeout(Duration::from_secs(2)).unwrap();
1258        let started = Instant::now();
1259        let result = wait(token, 10);
1260        let elapsed = started.elapsed();
1261        let observed = identity(token);
1262        let polled = poll(token);
1263        let _ = release_tx.send(());
1264        let terminal = owner.join().unwrap().expect("owner completes");
1265        close(token).expect("dispose");
1266        let snapshot = result.expect("bounded wait");
1267        assert!(
1268            elapsed < Duration::from_millis(300),
1269            "wait took {elapsed:?}"
1270        );
1271        assert_eq!(snapshot.handle_state, "active");
1272        assert_eq!(snapshot.leader_status, "running");
1273        assert_eq!(observed.unwrap().handle_state, "active");
1274        assert_eq!(polled.unwrap().handle_state, "active");
1275        assert_eq!(terminal.leader_status, "terminated");
1276    }
1277
1278    #[cfg(unix)]
1279    #[test]
1280    fn bounded_wait_during_active_close_preserves_close_owner() {
1281        let _lock = test_guard();
1282        let mut req = request(vec!["sleep", "30"]);
1283        req.grace_timeout_ms = Some(800);
1284        let token = spawn(req).expect("spawn owned child");
1285        let owner = thread::spawn(move || close(token));
1286        let (_, _, slot) = lookup(token).unwrap();
1287        let deadline = Instant::now() + Duration::from_secs(2);
1288        loop {
1289            let state = lock_data(slot).unwrap().state;
1290            if state != SlotState::Active || Instant::now() >= deadline {
1291                break;
1292            }
1293            thread::yield_now();
1294        }
1295        let started = Instant::now();
1296        let result = wait(token, 10);
1297        let elapsed = started.elapsed();
1298        owner.join().unwrap().expect("close owner completes");
1299        assert!(
1300            elapsed < Duration::from_millis(300),
1301            "wait took {elapsed:?}"
1302        );
1303        assert_eq!(result.unwrap().handle_state, "active");
1304        assert!(
1305            identity(token).is_err(),
1306            "close must recycle only after cleanup"
1307        );
1308    }
1309
1310    // Observe only cached state: poll/wait would themselves perform cleanup and
1311    // could hide a dead monitor. Always dispose before asserting the result.
1312    #[cfg(unix)]
1313    fn observe_native_finalization(token: u64) -> ManagedSnapshot {
1314        let deadline = Instant::now() + Duration::from_secs(3);
1315        loop {
1316            let snapshot = identity(token).expect("observe native monitor");
1317            if snapshot.handle_state == "inert" || Instant::now() >= deadline {
1318                close(token).expect("dispose monitor fixture");
1319                return snapshot;
1320            }
1321            thread::sleep(POLL_INTERVAL);
1322        }
1323    }
1324
1325    #[cfg(unix)]
1326    #[test]
1327    fn native_deadline_retries_unreaped_cleanup_without_language_wait() {
1328        let _lock = test_guard();
1329        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1330        let mut req = request(vec!["sleep", "30"]);
1331        req.execution_timeout_ms = Some(100);
1332        let token = spawn(req).expect("spawn with deadline");
1333        let snapshot = observe_native_finalization(token);
1334        assert!(!inject::FORCE_UNREAPED.load(Ordering::SeqCst));
1335        assert_eq!(snapshot.handle_state, "inert");
1336        assert_eq!(snapshot.leader_status, "timed_out");
1337        assert_eq!(snapshot.exited, Some(true));
1338    }
1339
1340    #[cfg(unix)]
1341    #[test]
1342    fn native_deadline_retries_operation_error_without_language_wait() {
1343        let _lock = test_guard();
1344        inject::FORCE_OPERATION_ERROR.store(true, Ordering::SeqCst);
1345        let mut req = request(vec!["sleep", "30"]);
1346        req.execution_timeout_ms = Some(100);
1347        let token = spawn(req).expect("spawn with deadline");
1348        let snapshot = observe_native_finalization(token);
1349        assert!(!inject::FORCE_OPERATION_ERROR.load(Ordering::SeqCst));
1350        assert_eq!(snapshot.handle_state, "inert");
1351        assert_eq!(snapshot.leader_status, "timed_out");
1352        assert_eq!(snapshot.exited, Some(true));
1353    }
1354
1355    #[cfg(unix)]
1356    #[test]
1357    fn native_deadline_preserves_unpolled_natural_completion() {
1358        let _lock = test_guard();
1359        let mut req = request(vec!["true"]);
1360        req.execution_timeout_ms = Some(500);
1361        let token = spawn(req).expect("spawn short-lived child with deadline");
1362        let snapshot = observe_native_finalization(token);
1363        assert_eq!(snapshot.handle_state, "inert");
1364        assert_eq!(snapshot.leader_status, "completed");
1365        assert_eq!(snapshot.exited, Some(true));
1366        assert_eq!(snapshot.timed_out, Some(false));
1367    }
1368
1369    #[cfg(unix)]
1370    #[test]
1371    fn deadline_reason_survives_failed_cleanup_and_explicit_retry() {
1372        let _lock = test_guard();
1373        let token = spawn(request(vec!["sleep", "30"])).expect("spawn deadline fixture");
1374        inject::FORCE_OPERATION_ERROR.store(true, Ordering::SeqCst);
1375        assert!(terminate_with_status(token, LeaderStatus::TimedOut).is_err());
1376        let active = identity(token).expect("failed deadline retains owner");
1377        let snapshot = terminate(token).expect("explicit retry completes cleanup");
1378        let cached = wait(token, 1).expect("wait returns cached outcome");
1379        close(token).expect("close finalized fixture");
1380        assert_eq!(active.handle_state, "active");
1381        assert_eq!(active.leader_status, "running");
1382        assert_eq!(snapshot.leader_status, "timed_out");
1383        assert_eq!(cached.leader_status, "timed_out");
1384    }
1385
1386    #[cfg(windows)]
1387    #[test]
1388    fn windows_spawn_fails_before_creating_a_handle() {
1389        let _lock = test_guard();
1390        let marker = std::env::temp_dir().join(format!(
1391            "sysprims-containment-windows-{}.marker",
1392            std::process::id()
1393        ));
1394        let _ = std::fs::remove_file(&marker);
1395        let result = spawn(request(vec![
1396            "cmd",
1397            "/C",
1398            &format!("echo spawned>{}", marker.display()),
1399        ]));
1400        assert!(result.is_err());
1401        assert!(!marker.exists(), "windows rejection must not start argv");
1402    }
1403
1404    #[cfg(unix)]
1405    #[test]
1406    fn invalid_high_signal_does_not_spawn() {
1407        let _lock = test_guard();
1408        let marker = std::env::temp_dir().join(format!(
1409            "sysprims-containment-high-signal-{}.marker",
1410            std::process::id()
1411        ));
1412        let _ = std::fs::remove_file(&marker);
1413        let mut req = request(vec!["touch", marker.to_str().expect("utf8 path")]);
1414        req.signal = Some(99);
1415        assert!(spawn(req).is_err());
1416        assert!(!marker.exists(), "invalid high signal must not start argv");
1417    }
1418
1419    #[cfg(unix)]
1420    #[test]
1421    fn failed_active_close_preserves_owner_and_unreaped_is_not_inert() {
1422        let _lock = test_guard();
1423        let token = spawn(request(vec!["sleep", "30"])).expect("spawn sleep");
1424
1425        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1426        let terminated = terminate(token);
1427        assert!(terminated.is_err(), "exited:false must be retryable");
1428        let snap = identity(token).expect("owner retained after unreaped terminate");
1429        assert_eq!(snap.handle_state, "active");
1430
1431        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1432        assert!(close(token).is_err(), "unconfirmed close must fail");
1433        identity(token).expect("failed close must keep the same generation");
1434        poll(token).expect("active owner remains usable");
1435
1436        inject::FORCE_UNREAPED.store(false, Ordering::SeqCst);
1437        close(token).expect("retry close after unreaped failure");
1438        assert!(identity(token).is_err());
1439    }
1440
1441    #[cfg(unix)]
1442    #[test]
1443    fn deadline_monitor_setup_failure_returns_no_handle() {
1444        let _lock = test_guard();
1445        let before = occupied_slots();
1446        let marker = std::env::temp_dir().join(format!(
1447            "sysprims-containment-deadline-setup-{}.marker",
1448            std::process::id()
1449        ));
1450        let _ = std::fs::remove_file(&marker);
1451        inject::FORCE_DEADLINE_THREAD_FAIL.store(true, Ordering::SeqCst);
1452        let mut req = request(vec!["touch", marker.to_str().expect("utf8 path")]);
1453        req.execution_timeout_ms = Some(5_000);
1454        let result = spawn(req);
1455        inject::FORCE_DEADLINE_THREAD_FAIL.store(false, Ordering::SeqCst);
1456        assert!(
1457            result.is_err(),
1458            "deadline setup failure must not return a handle"
1459        );
1460        assert!(
1461            !marker.exists(),
1462            "deadline reservation failure must not start argv"
1463        );
1464        assert_eq!(
1465            occupied_slots(),
1466            before,
1467            "monitor-setup failure before spawn must not occupy a slot"
1468        );
1469        let token = spawn(request(vec!["true"])).expect("registry remains usable");
1470        close(token).expect("close after deadline-setup failure");
1471    }
1472
1473    #[cfg(unix)]
1474    #[test]
1475    fn failed_active_close_keeps_native_deadline_live() {
1476        let _lock = test_guard();
1477        let mut req = request(vec!["sleep", "30"]);
1478        req.execution_timeout_ms = Some(250);
1479        req.grace_timeout_ms = Some(20);
1480        let token = spawn(req).expect("spawn with deadline");
1481
1482        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1483        assert!(close(token).is_err(), "unconfirmed close must fail");
1484        inject::FORCE_UNREAPED.store(false, Ordering::SeqCst);
1485
1486        let snap = identity(token).expect("owner retained after failed close");
1487        assert_eq!(snap.handle_state, "active");
1488        let snap = wait(token, 5_000).expect("native deadline must still fire");
1489        assert_eq!(snap.leader_status, "timed_out");
1490        close(token).expect("close after native timeout");
1491    }
1492
1493    #[cfg(unix)]
1494    #[test]
1495    fn concurrent_spawn_and_close_does_not_deadlock() {
1496        let _lock = test_guard();
1497        let (done_tx, done_rx) = mpsc::channel();
1498        let workers = 8;
1499        let iterations = 20;
1500        for _ in 0..workers {
1501            let done_tx = done_tx.clone();
1502            thread::spawn(move || {
1503                for _ in 0..iterations {
1504                    if let Ok(token) = spawn(request(vec!["true"])) {
1505                        let _ = wait(token, 2_000);
1506                        let _ = close(token);
1507                    }
1508                }
1509                let _ = done_tx.send(());
1510            });
1511        }
1512        drop(done_tx);
1513        for _ in 0..workers {
1514            done_rx
1515                .recv_timeout(Duration::from_secs(30))
1516                .expect("spawn/close worker deadlocked or timed out");
1517        }
1518    }
1519}