Skip to main content

sysprims_timeout/
managed.rs

1//! Native-owned managed containment lifecycle for language bindings.
2//!
3//! One registry owns every [`ContainmentGuard<Child>`] used by C/Go and N-API
4//! projections. Callers receive a generation-checked `u64` capability token,
5//! never a PID, Rust pointer, or reusable proof of spawn.
6
7use std::collections::BTreeMap;
8#[cfg(unix)]
9use std::process::{Command, Stdio};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock};
12use std::thread;
13use std::time::{Duration, Instant};
14
15use serde::Serialize;
16use sysprims_core::schema::CONTAINMENT_SNAPSHOT_V1;
17use sysprims_core::time::now_rfc3339;
18use sysprims_core::{get_platform, SysprimsError, SysprimsResult};
19
20use crate::{
21    ContainmentBoundaryStrength, ContainmentCompletionEvidence, ContainmentGuard,
22    ContainmentIdentity, ContainmentObservation, ContainmentOutcome, TerminateTreeConfig,
23    TreeKillReliability, SIGKILL, SIGTERM,
24};
25
26#[cfg(unix)]
27use crate::{spawn_contained, ContainmentSpawnError};
28
29pub const MAX_REGISTRY_SLOTS: usize = 1024;
30pub const MAX_ARGV_ENTRIES: usize = 256;
31pub const MAX_ARGV_ENTRY_BYTES: usize = 4096;
32pub const MAX_ARGV_TOTAL_BYTES: usize = 65_536;
33pub const MAX_ENV_ENTRIES: usize = 256;
34pub const MAX_ENV_KEY_BYTES: usize = 1024;
35pub const MAX_ENV_VALUE_BYTES: usize = 4096;
36pub const MAX_ENV_TOTAL_BYTES: usize = 65_536;
37pub const MAX_CWD_BYTES: usize = 4096;
38pub const MAX_DURATION_MS: u64 = 86_400_000;
39const POLL_INTERVAL: Duration = Duration::from_millis(10);
40
41#[derive(Debug, Clone)]
42pub struct ManagedSpawnRequest {
43    pub argv: Vec<String>,
44    pub cwd: Option<String>,
45    pub env: Option<BTreeMap<String, String>>,
46    pub execution_timeout_ms: Option<u64>,
47    pub grace_timeout_ms: Option<u64>,
48    pub kill_timeout_ms: Option<u64>,
49    pub signal: Option<i32>,
50    pub kill_signal: Option<i32>,
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54enum SlotState {
55    Empty,
56    Active,
57    Finalizing,
58    Inert,
59}
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62enum LeaderStatus {
63    Running,
64    Completed,
65    TimedOut,
66    Terminated,
67}
68
69impl LeaderStatus {
70    fn as_str(self) -> &'static str {
71        match self {
72            Self::Running => "running",
73            Self::Completed => "completed",
74            Self::TimedOut => "timed_out",
75            Self::Terminated => "terminated",
76        }
77    }
78}
79
80struct SlotData {
81    generation: u32,
82    retired: bool,
83    state: SlotState,
84    guard: Option<ContainmentGuard<ChildAdapter>>,
85    identity: Option<ContainmentIdentity>,
86    reliability: Option<TreeKillReliability>,
87    boundary: Option<ContainmentBoundaryStrength>,
88    terminate_config: TerminateTreeConfig,
89    outcome: Option<ContainmentOutcome>,
90    // Also records pending deadline enforcement while cleanup is retryable.
91    // Active snapshots still report Running until reap is confirmed.
92    leader_status: LeaderStatus,
93    deadline_cancel: Arc<AtomicBool>,
94    deadline_at: Option<Instant>,
95}
96
97struct Slot {
98    data: Mutex<SlotData>,
99    cond: Condvar,
100}
101
102struct Registry {
103    slots: Vec<Slot>,
104    free: Mutex<Vec<u32>>,
105}
106
107type ChildAdapter = std::process::Child;
108
109fn registry() -> &'static Registry {
110    static REGISTRY: OnceLock<Registry> = OnceLock::new();
111    REGISTRY.get_or_init(|| {
112        let mut slots = Vec::with_capacity(MAX_REGISTRY_SLOTS);
113        let mut free = Vec::with_capacity(MAX_REGISTRY_SLOTS);
114        for index in 0..MAX_REGISTRY_SLOTS {
115            slots.push(Slot {
116                data: Mutex::new(SlotData {
117                    generation: 1,
118                    retired: false,
119                    state: SlotState::Empty,
120                    guard: None,
121                    identity: None,
122                    reliability: None,
123                    boundary: None,
124                    terminate_config: TerminateTreeConfig::default(),
125                    outcome: None,
126                    leader_status: LeaderStatus::Running,
127                    deadline_cancel: Arc::new(AtomicBool::new(false)),
128                    deadline_at: None,
129                }),
130                cond: Condvar::new(),
131            });
132            free.push(index as u32);
133        }
134        Registry {
135            slots,
136            free: Mutex::new(free),
137        }
138    })
139}
140
141fn lock_poisoned() -> SysprimsError {
142    SysprimsError::internal("containment registry lock poisoned")
143}
144
145fn lock_data(slot: &Slot) -> SysprimsResult<MutexGuard<'_, SlotData>> {
146    slot.data.lock().map_err(|_| lock_poisoned())
147}
148
149fn lock_free(reg: &Registry) -> SysprimsResult<MutexGuard<'_, Vec<u32>>> {
150    reg.free.lock().map_err(|_| lock_poisoned())
151}
152
153#[cfg(unix)]
154fn pack_token(slot: u32, generation: u32) -> u64 {
155    ((generation as u64) << 32) | (u64::from(slot) + 1)
156}
157
158fn unpack_token(token: u64) -> SysprimsResult<(u32, u32)> {
159    if token == 0 {
160        return Err(invalid_handle("containment handle is invalid"));
161    }
162    let slot_plus_one = token & 0xffff_ffff;
163    if slot_plus_one == 0 {
164        return Err(invalid_handle("containment handle is invalid"));
165    }
166    let slot = slot_plus_one - 1;
167    if slot >= MAX_REGISTRY_SLOTS as u64 {
168        return Err(invalid_handle("containment handle is invalid"));
169    }
170    let generation = (token >> 32) as u32;
171    if generation == 0 {
172        return Err(invalid_handle("containment handle is invalid"));
173    }
174    Ok((slot as u32, generation))
175}
176
177fn invalid_handle(message: &str) -> SysprimsError {
178    SysprimsError::invalid_argument(message)
179}
180
181fn stale_handle() -> SysprimsError {
182    invalid_handle("containment handle is stale or closed")
183}
184
185#[cfg(unix)]
186fn spawn_error(error: ContainmentSpawnError) -> SysprimsError {
187    match error {
188        ContainmentSpawnError::Spawn(error) | ContainmentSpawnError::Adoption(error) => error,
189    }
190}
191
192pub fn validate_spawn_request(
193    request: &ManagedSpawnRequest,
194) -> SysprimsResult<TerminateTreeConfig> {
195    if request.argv.is_empty() {
196        return Err(SysprimsError::invalid_argument("argv must not be empty"));
197    }
198    if request.argv.len() > MAX_ARGV_ENTRIES {
199        return Err(SysprimsError::invalid_argument(format!(
200            "argv exceeds maximum of {MAX_ARGV_ENTRIES} entries"
201        )));
202    }
203    let mut argv_bytes = 0usize;
204    for (index, arg) in request.argv.iter().enumerate() {
205        if arg.is_empty() {
206            return Err(SysprimsError::invalid_argument(format!(
207                "argv[{index}] must not be empty"
208            )));
209        }
210        if arg.len() > MAX_ARGV_ENTRY_BYTES {
211            return Err(SysprimsError::invalid_argument(format!(
212                "argv[{index}] exceeds maximum of {MAX_ARGV_ENTRY_BYTES} bytes"
213            )));
214        }
215        argv_bytes = argv_bytes.saturating_add(arg.len());
216        if argv_bytes > MAX_ARGV_TOTAL_BYTES {
217            return Err(SysprimsError::invalid_argument(format!(
218                "argv exceeds maximum of {MAX_ARGV_TOTAL_BYTES} bytes"
219            )));
220        }
221    }
222
223    if let Some(cwd) = request.cwd.as_deref() {
224        if cwd.is_empty() {
225            return Err(SysprimsError::invalid_argument("cwd must not be empty"));
226        }
227        if cwd.len() > MAX_CWD_BYTES {
228            return Err(SysprimsError::invalid_argument(format!(
229                "cwd exceeds maximum of {MAX_CWD_BYTES} bytes"
230            )));
231        }
232    }
233
234    if let Some(env) = request.env.as_ref() {
235        if env.len() > MAX_ENV_ENTRIES {
236            return Err(SysprimsError::invalid_argument(format!(
237                "env exceeds maximum of {MAX_ENV_ENTRIES} entries"
238            )));
239        }
240        let mut env_bytes = 0usize;
241        for (key, value) in env {
242            if key.is_empty() {
243                return Err(SysprimsError::invalid_argument(
244                    "env keys must not be empty",
245                ));
246            }
247            if key.len() > MAX_ENV_KEY_BYTES {
248                return Err(SysprimsError::invalid_argument(format!(
249                    "env key exceeds maximum of {MAX_ENV_KEY_BYTES} bytes"
250                )));
251            }
252            if value.len() > MAX_ENV_VALUE_BYTES {
253                return Err(SysprimsError::invalid_argument(format!(
254                    "env value exceeds maximum of {MAX_ENV_VALUE_BYTES} bytes"
255                )));
256            }
257            env_bytes = env_bytes
258                .saturating_add(key.len())
259                .saturating_add(value.len());
260            if env_bytes > MAX_ENV_TOTAL_BYTES {
261                return Err(SysprimsError::invalid_argument(format!(
262                    "env exceeds maximum of {MAX_ENV_TOTAL_BYTES} bytes"
263                )));
264            }
265        }
266    }
267
268    if let Some(timeout) = request.execution_timeout_ms {
269        if timeout == 0 || timeout > MAX_DURATION_MS {
270            return Err(SysprimsError::invalid_argument(format!(
271                "execution_timeout_ms must be between 1 and {MAX_DURATION_MS}"
272            )));
273        }
274    }
275
276    let mut config = TerminateTreeConfig::default();
277    if let Some(grace) = request.grace_timeout_ms {
278        if grace > MAX_DURATION_MS {
279            return Err(SysprimsError::invalid_argument(format!(
280                "grace_timeout_ms exceeds maximum of {MAX_DURATION_MS}"
281            )));
282        }
283        config.grace_timeout_ms = grace;
284    }
285    if let Some(kill) = request.kill_timeout_ms {
286        if kill > MAX_DURATION_MS {
287            return Err(SysprimsError::invalid_argument(format!(
288                "kill_timeout_ms exceeds maximum of {MAX_DURATION_MS}"
289            )));
290        }
291        config.kill_timeout_ms = kill;
292    }
293    if let Some(signal) = request.signal {
294        validate_platform_signal(signal, "signal")?;
295        config.signal = signal;
296    }
297    if let Some(kill_signal) = request.kill_signal {
298        validate_platform_signal(kill_signal, "kill_signal")?;
299        config.kill_signal = kill_signal;
300    }
301    if config.signal == 0 {
302        config.signal = SIGTERM;
303    }
304    if config.kill_signal == 0 {
305        config.kill_signal = SIGKILL;
306    }
307    Ok(config)
308}
309
310fn platform_max_signal() -> i32 {
311    #[cfg(target_os = "linux")]
312    {
313        libc::SIGRTMAX()
314    }
315    #[cfg(not(target_os = "linux"))]
316    {
317        31
318    }
319}
320
321fn validate_platform_signal(signal: i32, name: &str) -> SysprimsResult<()> {
322    let max_signal = platform_max_signal();
323    if signal <= 0 || signal > max_signal {
324        return Err(SysprimsError::invalid_argument(format!(
325            "{name} must be between 1 and {max_signal}"
326        )));
327    }
328    Ok(())
329}
330
331fn cleanup_not_confirmed() -> SysprimsError {
332    SysprimsError::invalid_argument("containment cleanup did not confirm child reaped; retry")
333}
334
335#[cfg(unix)]
336fn allocate_slot() -> SysprimsResult<(u32, u32)> {
337    let reg = registry();
338    let mut free = lock_free(reg)?;
339    while let Some(slot_index) = free.pop() {
340        let slot = &reg.slots[slot_index as usize];
341        let data = lock_data(slot)?;
342        if data.retired || data.state != SlotState::Empty {
343            debug_assert!(
344                data.retired && data.state == SlotState::Empty,
345                "free list contained a live or inconsistent slot"
346            );
347            continue;
348        }
349        let generation = data.generation;
350        drop(data);
351        return Ok((slot_index, generation));
352    }
353    Err(SysprimsError::invalid_argument(
354        "containment registry has no free slots",
355    ))
356}
357
358/// Reset a slot to Empty and bump generation. Returns whether the index should
359/// be pushed onto the free list. Caller must drop the slot data lock before
360/// taking the free-list lock.
361fn reset_slot_for_reuse(data: &mut SlotData) -> bool {
362    data.guard = None;
363    data.identity = None;
364    data.reliability = None;
365    data.boundary = None;
366    data.outcome = None;
367    data.leader_status = LeaderStatus::Running;
368    data.deadline_cancel.store(true, Ordering::SeqCst);
369    data.deadline_cancel = Arc::new(AtomicBool::new(false));
370    data.deadline_at = None;
371    data.state = SlotState::Empty;
372    if data.generation == u32::MAX {
373        data.retired = true;
374        return false;
375    }
376    let next = data.generation.saturating_add(1);
377    if next == 0 || next == u32::MAX {
378        data.generation = u32::MAX;
379        data.retired = true;
380        return false;
381    }
382    data.generation = next;
383    true
384}
385
386fn push_free_index(slot_index: u32) -> SysprimsResult<()> {
387    let mut free = lock_free(registry())?;
388    free.push(slot_index);
389    Ok(())
390}
391
392fn recycle_after_data_unlock(
393    slot: &Slot,
394    slot_index: u32,
395    mut data: MutexGuard<'_, SlotData>,
396) -> SysprimsResult<()> {
397    let should_push = reset_slot_for_reuse(&mut data);
398    slot.cond.notify_all();
399    drop(data);
400    if should_push {
401        push_free_index(slot_index)?;
402    }
403    Ok(())
404}
405
406fn lookup<'a>(token: u64) -> SysprimsResult<(u32, u32, &'a Slot)> {
407    let (slot_index, generation) = unpack_token(token)?;
408    let slot = &registry().slots[slot_index as usize];
409    Ok((slot_index, generation, slot))
410}
411
412fn snapshot_from(
413    data: &SlotData,
414    handle_state: &'static str,
415    leader_status: LeaderStatus,
416) -> SysprimsResult<ManagedSnapshot> {
417    let identity = data
418        .identity
419        .as_ref()
420        .ok_or_else(|| SysprimsError::internal("containment identity missing"))?;
421    let reliability = data
422        .reliability
423        .ok_or_else(|| SysprimsError::internal("containment reliability missing"))?;
424    let boundary = data
425        .boundary
426        .ok_or_else(|| SysprimsError::internal("containment boundary missing"))?;
427    let mut snapshot = ManagedSnapshot {
428        schema_id: CONTAINMENT_SNAPSHOT_V1,
429        timestamp: now_rfc3339(),
430        platform: get_platform(),
431        handle_state,
432        leader_status: leader_status.as_str(),
433        identity: ManagedIdentity {
434            pid: identity.pid,
435            start_time_unix_ms: identity.start_time_unix_ms,
436            exe_path: identity.exe_path.clone(),
437        },
438        tree_kill_reliability: reliability.as_str(),
439        boundary_strength: boundary.as_str(),
440        pgid: None,
441        signal_sent: None,
442        kill_signal: None,
443        escalated: None,
444        exited: None,
445        timed_out: None,
446        completion: None,
447        warnings: Vec::new(),
448    };
449    if let Some(outcome) = data.outcome.as_ref() {
450        snapshot.pgid = outcome.pgid;
451        snapshot.signal_sent = outcome.signal_sent;
452        snapshot.kill_signal = outcome.kill_signal;
453        snapshot.escalated = Some(outcome.escalated);
454        snapshot.exited = Some(outcome.exited);
455        snapshot.timed_out = Some(outcome.timed_out);
456        snapshot.completion = Some(outcome.completion.clone());
457        snapshot.warnings = outcome.warnings.clone();
458    }
459    Ok(snapshot)
460}
461
462fn commit_inert(
463    slot: &Slot,
464    data: &mut SlotData,
465    outcome: ContainmentOutcome,
466    leader_status: LeaderStatus,
467) -> SysprimsResult<ManagedSnapshot> {
468    data.deadline_cancel.store(true, Ordering::SeqCst);
469    data.outcome = Some(outcome);
470    data.leader_status = leader_status;
471    data.state = SlotState::Inert;
472    slot.cond.notify_all();
473    snapshot_from(data, "inert", leader_status)
474}
475
476/// Spawn a sysprims-owned contained child and return a capability token.
477pub fn spawn(request: ManagedSpawnRequest) -> SysprimsResult<u64> {
478    let terminate_config = validate_spawn_request(&request)?;
479
480    #[cfg(windows)]
481    {
482        let _ = terminate_config;
483        return Err(SysprimsError::not_supported(
484            "managed containment spawn without create-suspended Job assignment",
485            "windows",
486        ));
487    }
488
489    #[cfg(unix)]
490    {
491        let (slot_index, generation) = allocate_slot()?;
492        let token = pack_token(slot_index, generation);
493        let pending_deadline = match request.execution_timeout_ms {
494            Some(timeout_ms) => match start_unarmed_deadline_monitor() {
495                Ok(monitor) => Some((timeout_ms, monitor)),
496                Err(error) => {
497                    let _ = recycle_unused_slot(slot_index, generation);
498                    return Err(error);
499                }
500            },
501            None => None,
502        };
503
504        let command = match build_command(&request) {
505            Ok(command) => command,
506            Err(error) => {
507                abandon_unused_construction(slot_index, generation, pending_deadline.as_ref());
508                return Err(error);
509            }
510        };
511
512        let guard = match spawn_contained(command) {
513            Ok(guard) => guard,
514            Err(error) => {
515                abandon_unused_construction(slot_index, generation, pending_deadline.as_ref());
516                return Err(spawn_error(error));
517            }
518        };
519
520        let reliability = guard.tree_kill_reliability();
521        let boundary = guard.boundary_strength();
522        if reliability != TreeKillReliability::Guaranteed
523            || boundary != ContainmentBoundaryStrength::CooperativeGroup
524        {
525            drop(guard);
526            abandon_unused_construction(slot_index, generation, pending_deadline.as_ref());
527            return Err(SysprimsError::internal(
528                "managed spawn produced a non-guaranteed cooperative-group guard",
529            ));
530        }
531
532        let identity = guard.identity().clone();
533        let slot = &registry().slots[slot_index as usize];
534        let deadline_at = pending_deadline
535            .as_ref()
536            .map(|(timeout_ms, _)| Instant::now() + Duration::from_millis(*timeout_ms));
537        {
538            let mut data = lock_data(slot)?;
539            if data.generation != generation || data.state != SlotState::Empty {
540                drop(guard);
541                if let Some((_, monitor)) = pending_deadline.as_ref() {
542                    monitor.cancel();
543                }
544                return Err(SysprimsError::internal(
545                    "containment slot changed before spawn insert",
546                ));
547            }
548            data.identity = Some(identity);
549            data.reliability = Some(reliability);
550            data.boundary = Some(boundary);
551            data.terminate_config = terminate_config;
552            data.leader_status = LeaderStatus::Running;
553            data.outcome = None;
554            if let Some((_, monitor)) = pending_deadline.as_ref() {
555                data.deadline_cancel = Arc::clone(&monitor.cancel);
556            }
557            data.deadline_at = deadline_at;
558            data.guard = Some(guard);
559            data.state = SlotState::Active;
560        }
561        if let Some((_, monitor)) = pending_deadline {
562            if let Some(deadline) = deadline_at {
563                monitor.arm(token, deadline);
564            }
565        }
566        Ok(token)
567    }
568}
569
570#[cfg(unix)]
571fn abandon_unused_construction(
572    slot_index: u32,
573    generation: u32,
574    pending_deadline: Option<&(u64, DeadlineMonitor)>,
575) {
576    if let Some((_, monitor)) = pending_deadline {
577        monitor.cancel();
578    }
579    let _ = recycle_unused_slot(slot_index, generation);
580}
581
582#[cfg(unix)]
583fn recycle_unused_slot(slot_index: u32, generation: u32) -> SysprimsResult<()> {
584    let reg = registry();
585    let slot = &reg.slots[slot_index as usize];
586    let data = lock_data(slot)?;
587    if data.generation == generation && data.state == SlotState::Empty && !data.retired {
588        drop(data);
589        push_free_index(slot_index)?;
590    }
591    Ok(())
592}
593
594#[cfg(unix)]
595fn build_command(request: &ManagedSpawnRequest) -> SysprimsResult<Command> {
596    let mut command = Command::new(&request.argv[0]);
597    if request.argv.len() > 1 {
598        command.args(&request.argv[1..]);
599    }
600    command
601        .stdin(Stdio::null())
602        .stdout(Stdio::null())
603        .stderr(Stdio::null());
604    if let Some(cwd) = request.cwd.as_deref() {
605        command.current_dir(cwd);
606    }
607    if let Some(env) = request.env.as_ref() {
608        for (key, value) in env {
609            command.env(key, value);
610        }
611    }
612    Ok(command)
613}
614
615#[derive(Clone, Copy)]
616struct DeadlineArm {
617    token: u64,
618    deadline: Instant,
619}
620
621#[cfg(unix)]
622struct DeadlineMonitor {
623    arm: Arc<Mutex<Option<DeadlineArm>>>,
624    cancel: Arc<AtomicBool>,
625}
626
627#[cfg(unix)]
628impl DeadlineMonitor {
629    fn cancel(&self) {
630        self.cancel.store(true, Ordering::SeqCst);
631    }
632
633    /// Arm is infallible: the monitor thread is already reserved, and this
634    /// mutex has no panicking critical section. Recover poison and own the
635    /// arm state so a returned handle always has a deadline.
636    fn arm(&self, token: u64, deadline: Instant) {
637        *lock_arm(&self.arm) = Some(DeadlineArm { token, deadline });
638    }
639}
640
641fn lock_arm(arm: &Mutex<Option<DeadlineArm>>) -> MutexGuard<'_, Option<DeadlineArm>> {
642    arm.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
643}
644
645#[cfg(unix)]
646fn start_unarmed_deadline_monitor() -> SysprimsResult<DeadlineMonitor> {
647    if test_force_deadline_thread_fail() {
648        return Err(SysprimsError::system(
649            "failed to start containment deadline monitor",
650            0,
651        ));
652    }
653    let arm = Arc::new(Mutex::new(None));
654    let cancel = Arc::new(AtomicBool::new(false));
655    spawn_deadline_worker(Arc::clone(&arm), Arc::clone(&cancel))?;
656    Ok(DeadlineMonitor { arm, cancel })
657}
658
659fn spawn_deadline_worker(
660    arm: Arc<Mutex<Option<DeadlineArm>>>,
661    cancel: Arc<AtomicBool>,
662) -> SysprimsResult<()> {
663    thread::Builder::new()
664        .name("sysprims-containment-deadline".into())
665        .spawn(move || run_deadline_monitor(arm, cancel))
666        .map_err(|error| {
667            SysprimsError::system(
668                format!("failed to start containment deadline monitor: {error}"),
669                error.raw_os_error().unwrap_or(0),
670            )
671        })?;
672    Ok(())
673}
674
675fn run_deadline_monitor(arm: Arc<Mutex<Option<DeadlineArm>>>, cancel: Arc<AtomicBool>) {
676    loop {
677        if cancel.load(Ordering::SeqCst) {
678            return;
679        }
680        let armed = *lock_arm(&arm);
681        match armed {
682            Some(DeadlineArm { token, deadline }) => {
683                let now = Instant::now();
684                if now >= deadline {
685                    if terminate_with_status(token, LeaderStatus::TimedOut).is_ok() {
686                        return;
687                    }
688                    // Errors and unconfirmed reap retain the active capability.
689                    // Keep its existing monitor alive until finalization or close
690                    // cancels it; retry against the same generation-bound token.
691                    thread::sleep(POLL_INTERVAL);
692                    continue;
693                }
694                thread::sleep(POLL_INTERVAL.min(deadline.saturating_duration_since(now)));
695            }
696            None => thread::sleep(POLL_INTERVAL),
697        }
698    }
699}
700
701fn restore_deadline_monitor(data: &mut SlotData, token: u64) {
702    let Some(deadline) = data.deadline_at else {
703        return;
704    };
705    if !data.deadline_cancel.load(Ordering::SeqCst) {
706        return;
707    }
708    let cancel = Arc::new(AtomicBool::new(false));
709    let arm = Arc::new(Mutex::new(Some(DeadlineArm { token, deadline })));
710    if spawn_deadline_worker(Arc::clone(&arm), Arc::clone(&cancel)).is_ok() {
711        data.deadline_cancel = cancel;
712    }
713}
714
715#[cfg(unix)]
716fn test_force_deadline_thread_fail() -> bool {
717    #[cfg(test)]
718    {
719        inject::FORCE_DEADLINE_THREAD_FAIL.swap(false, Ordering::SeqCst)
720    }
721    #[cfg(not(test))]
722    {
723        false
724    }
725}
726
727fn test_force_unreaped() -> bool {
728    #[cfg(test)]
729    {
730        inject::FORCE_UNREAPED.swap(false, Ordering::SeqCst)
731    }
732    #[cfg(not(test))]
733    {
734        false
735    }
736}
737
738fn test_force_operation_error() -> bool {
739    #[cfg(test)]
740    {
741        inject::FORCE_OPERATION_ERROR.swap(false, Ordering::SeqCst)
742    }
743    #[cfg(not(test))]
744    {
745        false
746    }
747}
748
749#[cfg(test)]
750mod inject {
751    use std::sync::atomic::AtomicBool;
752    use std::sync::Mutex;
753
754    pub static FORCE_UNREAPED: AtomicBool = AtomicBool::new(false);
755    pub static FORCE_OPERATION_ERROR: AtomicBool = AtomicBool::new(false);
756    pub static FORCE_DEADLINE_THREAD_FAIL: AtomicBool = AtomicBool::new(false);
757    pub static TEST_MUTEX: Mutex<()> = Mutex::new(());
758}
759
760fn unreaped_outcome(data: &SlotData) -> SysprimsResult<ContainmentOutcome> {
761    let identity = data
762        .identity
763        .clone()
764        .ok_or_else(|| SysprimsError::internal("containment identity missing"))?;
765    let reliability = data
766        .reliability
767        .ok_or_else(|| SysprimsError::internal("containment reliability missing"))?;
768    let boundary = data
769        .boundary
770        .ok_or_else(|| SysprimsError::internal("containment boundary missing"))?;
771    Ok(ContainmentOutcome {
772        identity,
773        pgid: None,
774        signal_sent: None,
775        kill_signal: None,
776        escalated: false,
777        exited: false,
778        timed_out: true,
779        tree_kill_reliability: reliability,
780        boundary_strength: boundary,
781        completion: ContainmentCompletionEvidence::Unknown {
782            observation: ContainmentObservation::UnsupportedPlatform,
783        },
784        warnings: vec!["Timed out waiting to reap contained child".to_string()],
785    })
786}
787
788fn wait_for_not_finalizing<'a>(
789    slot: &'a Slot,
790    mut data: MutexGuard<'a, SlotData>,
791    wait_deadline: Option<Instant>,
792) -> SysprimsResult<MutexGuard<'a, SlotData>> {
793    let generation = data.generation;
794    while data.state == SlotState::Finalizing && data.generation == generation {
795        if let Some(deadline) = wait_deadline {
796            let now = Instant::now();
797            if now >= deadline {
798                return Ok(data);
799            }
800            let remaining = deadline.saturating_duration_since(now);
801            let (guard, _) = slot
802                .cond
803                .wait_timeout(data, remaining)
804                .map_err(|_| lock_poisoned())?;
805            data = guard;
806        } else {
807            data = slot.cond.wait(data).map_err(|_| lock_poisoned())?;
808        }
809    }
810    Ok(data)
811}
812
813pub fn identity(token: u64) -> SysprimsResult<ManagedSnapshot> {
814    let (_slot_index, generation, slot) = lookup(token)?;
815    let data = lock_data(slot)?;
816    if data.generation != generation || data.retired {
817        return Err(stale_handle());
818    }
819    match data.state {
820        SlotState::Empty => Err(stale_handle()),
821        SlotState::Active | SlotState::Finalizing => {
822            snapshot_from(&data, "active", LeaderStatus::Running)
823        }
824        SlotState::Inert => snapshot_from(&data, "inert", data.leader_status),
825    }
826}
827
828pub fn poll(token: u64) -> SysprimsResult<ManagedSnapshot> {
829    operate_non_blocking(
830        token,
831        LeaderStatus::Completed,
832        Some(Instant::now()),
833        |guard, config| guard.try_complete(config),
834    )
835}
836
837pub fn wait(token: u64, wait_timeout_ms: u64) -> SysprimsResult<ManagedSnapshot> {
838    if wait_timeout_ms > MAX_DURATION_MS {
839        return Err(SysprimsError::invalid_argument(format!(
840            "wait timeout exceeds maximum of {MAX_DURATION_MS}"
841        )));
842    }
843    let wait_deadline = if wait_timeout_ms == 0 {
844        None
845    } else {
846        Some(Instant::now() + Duration::from_millis(wait_timeout_ms))
847    };
848
849    loop {
850        match operate_non_blocking(
851            token,
852            LeaderStatus::Completed,
853            wait_deadline,
854            |guard, config| guard.try_complete(config),
855        ) {
856            Ok(snapshot) if snapshot.leader_status != "running" => return Ok(snapshot),
857            Ok(snapshot) => {
858                if let Some(deadline) = wait_deadline {
859                    if Instant::now() >= deadline {
860                        return Ok(snapshot);
861                    }
862                }
863                let remaining = wait_deadline
864                    .map(|deadline| deadline.saturating_duration_since(Instant::now()));
865                thread::sleep(
866                    remaining.map_or(POLL_INTERVAL, |remaining| POLL_INTERVAL.min(remaining)),
867                );
868            }
869            Err(error) => return Err(error),
870        }
871    }
872}
873
874pub fn terminate(token: u64) -> SysprimsResult<ManagedSnapshot> {
875    terminate_with_status(token, LeaderStatus::Terminated)
876}
877
878fn terminate_with_status(
879    token: u64,
880    leader_status: LeaderStatus,
881) -> SysprimsResult<ManagedSnapshot> {
882    operate_blocking(token, leader_status, |guard, config| {
883        guard.terminate(config).map(Some)
884    })
885}
886
887pub fn close(token: u64) -> SysprimsResult<()> {
888    let (slot_index, generation, slot) = lookup(token)?;
889    let mut data = lock_data(slot)?;
890    if data.generation != generation || data.retired {
891        return Err(stale_handle());
892    }
893    data = wait_for_not_finalizing(slot, data, Some(Instant::now() + Duration::from_secs(120)))?;
894    if data.generation != generation {
895        return Err(stale_handle());
896    }
897    match data.state {
898        SlotState::Empty => Err(stale_handle()),
899        SlotState::Finalizing => Err(SysprimsError::invalid_argument(
900            "containment handle is busy; retry close",
901        )),
902        SlotState::Inert => recycle_after_data_unlock(slot, slot_index, data),
903        SlotState::Active => {
904            let mut guard = data
905                .guard
906                .take()
907                .ok_or_else(|| SysprimsError::internal("containment guard missing during close"))?;
908            let config = data.terminate_config.clone();
909            let forced = test_force_unreaped().then(|| unreaped_outcome(&data));
910            data.state = SlotState::Finalizing;
911            drop(data);
912            // Finalizing reserves this generation; only this owner holds the
913            // guard. Release the state lock so bounded waiters can observe it.
914            let result = match forced {
915                Some(outcome) => outcome.map(Some),
916                None => guard.terminate(config).map(Some),
917            };
918            let mut data = lock_data(slot)?;
919            data.guard = Some(guard);
920            match result {
921                Ok(Some(outcome)) if outcome.exited => {
922                    recycle_after_data_unlock(slot, slot_index, data)
923                }
924                Ok(_) => {
925                    data.state = SlotState::Active;
926                    restore_deadline_monitor(&mut data, token);
927                    slot.cond.notify_all();
928                    Err(cleanup_not_confirmed())
929                }
930                Err(error) => {
931                    data.state = SlotState::Active;
932                    restore_deadline_monitor(&mut data, token);
933                    slot.cond.notify_all();
934                    Err(error)
935                }
936            }
937        }
938    }
939}
940
941fn operate_non_blocking(
942    token: u64,
943    complete_status: LeaderStatus,
944    wait_deadline: Option<Instant>,
945    op: impl FnOnce(
946        &mut ContainmentGuard<ChildAdapter>,
947        TerminateTreeConfig,
948    ) -> SysprimsResult<Option<ContainmentOutcome>>,
949) -> SysprimsResult<ManagedSnapshot> {
950    operate_inner(token, complete_status, wait_deadline, op)
951}
952
953fn operate_blocking(
954    token: u64,
955    complete_status: LeaderStatus,
956    op: impl FnOnce(
957        &mut ContainmentGuard<ChildAdapter>,
958        TerminateTreeConfig,
959    ) -> SysprimsResult<Option<ContainmentOutcome>>,
960) -> SysprimsResult<ManagedSnapshot> {
961    operate_inner(token, complete_status, None, op)
962}
963
964fn operate_inner(
965    token: u64,
966    mut complete_status: LeaderStatus,
967    wait_deadline: Option<Instant>,
968    op: impl FnOnce(
969        &mut ContainmentGuard<ChildAdapter>,
970        TerminateTreeConfig,
971    ) -> SysprimsResult<Option<ContainmentOutcome>>,
972) -> SysprimsResult<ManagedSnapshot> {
973    let (_slot_index, generation, slot) = lookup(token)?;
974    let mut data = lock_data(slot)?;
975    if data.generation != generation || data.retired {
976        return Err(stale_handle());
977    }
978    let waited_for_owner = data.state == SlotState::Finalizing;
979    data = wait_for_not_finalizing(slot, data, wait_deadline)?;
980    if data.generation != generation {
981        return Err(stale_handle());
982    }
983    // A failed owner may restore Active as this wait expires. Do not acquire
984    // cleanup ownership after spending the caller's budget waiting for it.
985    if waited_for_owner
986        && data.state == SlotState::Active
987        && wait_deadline.is_some_and(|deadline| Instant::now() >= deadline)
988    {
989        return snapshot_from(&data, "active", LeaderStatus::Running);
990    }
991    match data.state {
992        SlotState::Empty => Err(stale_handle()),
993        SlotState::Finalizing => snapshot_from(&data, "active", LeaderStatus::Running),
994        SlotState::Inert => snapshot_from(&data, "inert", data.leader_status),
995        SlotState::Active => {
996            let mut guard = data.guard.take().ok_or_else(|| {
997                SysprimsError::internal("containment guard missing during operation")
998            })?;
999            let config = data.terminate_config.clone();
1000            let forced = test_force_unreaped().then(|| unreaped_outcome(&data));
1001            let mut pending_status = data.leader_status;
1002            data.state = SlotState::Finalizing;
1003            drop(data);
1004            // This owner exclusively holds the guard until it restores the
1005            // slot. Other operations may observe/wait, but cannot recycle it.
1006            let result = (|| {
1007                if complete_status == LeaderStatus::TimedOut
1008                    && pending_status != LeaderStatus::TimedOut
1009                {
1010                    if let Some(outcome) = guard.try_complete(config.clone())? {
1011                        complete_status = LeaderStatus::Completed;
1012                        return Ok(Some(outcome));
1013                    }
1014                    pending_status = LeaderStatus::TimedOut;
1015                }
1016                if test_force_operation_error() {
1017                    return Err(SysprimsError::internal(
1018                        "injected containment operation failure",
1019                    ));
1020                }
1021                if let Some(outcome) = forced {
1022                    return outcome.map(Some);
1023                }
1024                op(&mut guard, config)
1025            })();
1026            let mut data = lock_data(slot)?;
1027            data.guard = Some(guard);
1028            data.leader_status = pending_status;
1029            match result {
1030                Ok(Some(outcome)) if outcome.exited => {
1031                    if data.leader_status == LeaderStatus::TimedOut {
1032                        complete_status = LeaderStatus::TimedOut;
1033                    }
1034                    commit_inert(slot, &mut data, outcome, complete_status)
1035                }
1036                Ok(Some(_outcome)) => {
1037                    data.state = SlotState::Active;
1038                    slot.cond.notify_all();
1039                    Err(cleanup_not_confirmed())
1040                }
1041                Ok(None) => {
1042                    data.state = SlotState::Active;
1043                    slot.cond.notify_all();
1044                    snapshot_from(&data, "active", LeaderStatus::Running)
1045                }
1046                Err(error) => {
1047                    data.state = SlotState::Active;
1048                    slot.cond.notify_all();
1049                    Err(error)
1050                }
1051            }
1052        }
1053    }
1054}
1055
1056#[derive(Debug, Clone, Serialize)]
1057pub struct ManagedIdentity {
1058    pub pid: u32,
1059    pub start_time_unix_ms: u64,
1060    pub exe_path: String,
1061}
1062
1063#[derive(Debug, Clone, Serialize)]
1064pub struct ManagedSnapshot {
1065    pub schema_id: &'static str,
1066    pub timestamp: String,
1067    pub platform: &'static str,
1068    pub handle_state: &'static str,
1069    pub leader_status: &'static str,
1070    pub identity: ManagedIdentity,
1071    pub tree_kill_reliability: &'static str,
1072    pub boundary_strength: &'static str,
1073    #[serde(skip_serializing_if = "Option::is_none")]
1074    pub pgid: Option<u32>,
1075    #[serde(skip_serializing_if = "Option::is_none")]
1076    pub signal_sent: Option<i32>,
1077    #[serde(skip_serializing_if = "Option::is_none")]
1078    pub kill_signal: Option<i32>,
1079    #[serde(skip_serializing_if = "Option::is_none")]
1080    pub escalated: Option<bool>,
1081    #[serde(skip_serializing_if = "Option::is_none")]
1082    pub exited: Option<bool>,
1083    #[serde(skip_serializing_if = "Option::is_none")]
1084    pub timed_out: Option<bool>,
1085    #[serde(skip_serializing_if = "Option::is_none")]
1086    pub completion: Option<ContainmentCompletionEvidence>,
1087    pub warnings: Vec<String>,
1088}
1089
1090#[cfg(test)]
1091mod tests {
1092    use super::*;
1093    use std::sync::atomic::Ordering;
1094    #[cfg(unix)]
1095    use std::sync::mpsc;
1096    use std::sync::MutexGuard;
1097    #[cfg(unix)]
1098    use std::thread;
1099
1100    fn test_guard() -> MutexGuard<'static, ()> {
1101        let guard = inject::TEST_MUTEX
1102            .lock()
1103            .unwrap_or_else(|poisoned| poisoned.into_inner());
1104        inject::FORCE_UNREAPED.store(false, Ordering::SeqCst);
1105        inject::FORCE_OPERATION_ERROR.store(false, Ordering::SeqCst);
1106        inject::FORCE_DEADLINE_THREAD_FAIL.store(false, Ordering::SeqCst);
1107        guard
1108    }
1109
1110    #[cfg(unix)]
1111    fn occupied_slots() -> usize {
1112        registry()
1113            .slots
1114            .iter()
1115            .filter(|slot| {
1116                lock_data(slot)
1117                    .map(|data| data.state != SlotState::Empty)
1118                    .unwrap_or(true)
1119            })
1120            .count()
1121    }
1122
1123    fn request(argv: Vec<&str>) -> ManagedSpawnRequest {
1124        ManagedSpawnRequest {
1125            argv: argv.into_iter().map(str::to_string).collect(),
1126            cwd: None,
1127            env: None,
1128            execution_timeout_ms: None,
1129            grace_timeout_ms: Some(50),
1130            kill_timeout_ms: Some(200),
1131            signal: None,
1132            kill_signal: None,
1133        }
1134    }
1135
1136    #[test]
1137    fn rejects_empty_argv_and_signal_zero() {
1138        let _lock = test_guard();
1139        assert!(validate_spawn_request(&request(vec![])).is_err());
1140        let mut bad = request(vec!["true"]);
1141        bad.signal = Some(0);
1142        assert!(validate_spawn_request(&bad).is_err());
1143        let mut overflow = request(vec!["true"]);
1144        overflow.execution_timeout_ms = Some(MAX_DURATION_MS + 1);
1145        assert!(validate_spawn_request(&overflow).is_err());
1146        let mut high = request(vec!["true"]);
1147        high.signal = Some(99);
1148        assert!(validate_spawn_request(&high).is_err());
1149        let mut high_kill = request(vec!["true"]);
1150        high_kill.kill_signal = Some(99);
1151        assert!(validate_spawn_request(&high_kill).is_err());
1152    }
1153
1154    #[test]
1155    fn unpack_rejects_zero_and_malformed_tokens() {
1156        let _lock = test_guard();
1157        assert!(unpack_token(0).is_err());
1158        assert!(unpack_token(1u64 << 32).is_err());
1159        assert!(identity(0).is_err());
1160        assert!(identity(1).is_err());
1161        assert!(identity(u64::MAX).is_err());
1162        assert!(close(0).is_err());
1163    }
1164
1165    #[cfg(unix)]
1166    #[test]
1167    fn spawn_wait_complete_dispose_twice_and_stale_reuse() {
1168        let _lock = test_guard();
1169        let token = spawn(request(vec!["true"])).expect("spawn true");
1170        let snap = wait(token, 5_000).expect("wait true");
1171        assert_eq!(snap.leader_status, "completed");
1172        assert_eq!(snap.tree_kill_reliability, "guaranteed");
1173        assert_eq!(snap.boundary_strength, "cooperative_group");
1174        assert_eq!(snap.handle_state, "inert");
1175        match snap.completion {
1176            Some(ContainmentCompletionEvidence::Empty { .. })
1177            | Some(ContainmentCompletionEvidence::Unknown { .. }) => {}
1178            other => panic!("unexpected completion: {other:?}"),
1179        }
1180        close(token).expect("close");
1181        assert!(
1182            close(token).is_err(),
1183            "closed token must fail closed at FFI"
1184        );
1185        let token2 = spawn(request(vec!["true"])).expect("reuse slot");
1186        assert_ne!(token, token2);
1187        assert!(
1188            identity(token).is_err(),
1189            "stale token must not follow slot reuse"
1190        );
1191        close(token2).ok();
1192    }
1193
1194    #[cfg(unix)]
1195    #[test]
1196    fn terminate_keeps_spawn_reliability_and_linearizes_waiters() {
1197        let _lock = test_guard();
1198        let token = spawn(request(vec!["sleep", "30"])).expect("spawn sleep");
1199        let identity_before = identity(token).expect("identity");
1200        let snap = terminate(token).expect("terminate");
1201        assert_eq!(snap.leader_status, "terminated");
1202        assert_eq!(
1203            snap.tree_kill_reliability,
1204            identity_before.tree_kill_reliability
1205        );
1206        let again = terminate(token).expect("inert terminate");
1207        assert_eq!(again.handle_state, "inert");
1208        assert_eq!(again.leader_status, "terminated");
1209        close(token).expect("close after terminate");
1210    }
1211
1212    #[cfg(unix)]
1213    #[test]
1214    fn native_deadline_times_out_without_language_wait() {
1215        let _lock = test_guard();
1216        let mut req = request(vec!["sleep", "30"]);
1217        req.execution_timeout_ms = Some(200);
1218        req.grace_timeout_ms = Some(20);
1219        let token = spawn(req).expect("spawn with deadline");
1220        let snap = wait(token, 5_000).expect("wait for native deadline");
1221        assert_eq!(snap.leader_status, "timed_out");
1222        assert_eq!(snap.tree_kill_reliability, "guaranteed");
1223        close(token).ok();
1224    }
1225
1226    #[cfg(unix)]
1227    #[test]
1228    fn bounded_wait_does_not_wait_for_finalizing_owner() {
1229        let _lock = test_guard();
1230        let token = spawn(request(vec!["sleep", "30"])).expect("spawn owned child");
1231        let (entered_tx, entered_rx) = mpsc::channel();
1232        let (release_tx, release_rx) = mpsc::channel();
1233        let owner = thread::spawn(move || {
1234            operate_blocking(token, LeaderStatus::Terminated, |guard, config| {
1235                entered_tx.send(()).unwrap();
1236                // Bound the fixture even if the waiter regresses and blocks.
1237                let _ = release_rx.recv_timeout(Duration::from_secs(1));
1238                guard.terminate(config).map(Some)
1239            })
1240        });
1241        entered_rx.recv_timeout(Duration::from_secs(2)).unwrap();
1242        let started = Instant::now();
1243        let result = wait(token, 10);
1244        let elapsed = started.elapsed();
1245        let observed = identity(token);
1246        let polled = poll(token);
1247        let _ = release_tx.send(());
1248        let terminal = owner.join().unwrap().expect("owner completes");
1249        close(token).expect("dispose");
1250        let snapshot = result.expect("bounded wait");
1251        assert!(
1252            elapsed < Duration::from_millis(300),
1253            "wait took {elapsed:?}"
1254        );
1255        assert_eq!(snapshot.handle_state, "active");
1256        assert_eq!(snapshot.leader_status, "running");
1257        assert_eq!(observed.unwrap().handle_state, "active");
1258        assert_eq!(polled.unwrap().handle_state, "active");
1259        assert_eq!(terminal.leader_status, "terminated");
1260    }
1261
1262    #[cfg(unix)]
1263    #[test]
1264    fn bounded_wait_during_active_close_preserves_close_owner() {
1265        let _lock = test_guard();
1266        let mut req = request(vec!["sleep", "30"]);
1267        req.grace_timeout_ms = Some(800);
1268        let token = spawn(req).expect("spawn owned child");
1269        let owner = thread::spawn(move || close(token));
1270        let (_, _, slot) = lookup(token).unwrap();
1271        let deadline = Instant::now() + Duration::from_secs(2);
1272        loop {
1273            let state = lock_data(slot).unwrap().state;
1274            if state != SlotState::Active || Instant::now() >= deadline {
1275                break;
1276            }
1277            thread::yield_now();
1278        }
1279        let started = Instant::now();
1280        let result = wait(token, 10);
1281        let elapsed = started.elapsed();
1282        owner.join().unwrap().expect("close owner completes");
1283        assert!(
1284            elapsed < Duration::from_millis(300),
1285            "wait took {elapsed:?}"
1286        );
1287        assert_eq!(result.unwrap().handle_state, "active");
1288        assert!(
1289            identity(token).is_err(),
1290            "close must recycle only after cleanup"
1291        );
1292    }
1293
1294    // Observe only cached state: poll/wait would themselves perform cleanup and
1295    // could hide a dead monitor. Always dispose before asserting the result.
1296    #[cfg(unix)]
1297    fn observe_native_finalization(token: u64) -> ManagedSnapshot {
1298        let deadline = Instant::now() + Duration::from_secs(3);
1299        loop {
1300            let snapshot = identity(token).expect("observe native monitor");
1301            if snapshot.handle_state == "inert" || Instant::now() >= deadline {
1302                close(token).expect("dispose monitor fixture");
1303                return snapshot;
1304            }
1305            thread::sleep(POLL_INTERVAL);
1306        }
1307    }
1308
1309    #[cfg(unix)]
1310    #[test]
1311    fn native_deadline_retries_unreaped_cleanup_without_language_wait() {
1312        let _lock = test_guard();
1313        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1314        let mut req = request(vec!["sleep", "30"]);
1315        req.execution_timeout_ms = Some(100);
1316        let token = spawn(req).expect("spawn with deadline");
1317        let snapshot = observe_native_finalization(token);
1318        assert!(!inject::FORCE_UNREAPED.load(Ordering::SeqCst));
1319        assert_eq!(snapshot.handle_state, "inert");
1320        assert_eq!(snapshot.leader_status, "timed_out");
1321        assert_eq!(snapshot.exited, Some(true));
1322    }
1323
1324    #[cfg(unix)]
1325    #[test]
1326    fn native_deadline_retries_operation_error_without_language_wait() {
1327        let _lock = test_guard();
1328        inject::FORCE_OPERATION_ERROR.store(true, Ordering::SeqCst);
1329        let mut req = request(vec!["sleep", "30"]);
1330        req.execution_timeout_ms = Some(100);
1331        let token = spawn(req).expect("spawn with deadline");
1332        let snapshot = observe_native_finalization(token);
1333        assert!(!inject::FORCE_OPERATION_ERROR.load(Ordering::SeqCst));
1334        assert_eq!(snapshot.handle_state, "inert");
1335        assert_eq!(snapshot.leader_status, "timed_out");
1336        assert_eq!(snapshot.exited, Some(true));
1337    }
1338
1339    #[cfg(unix)]
1340    #[test]
1341    fn native_deadline_preserves_unpolled_natural_completion() {
1342        let _lock = test_guard();
1343        let mut req = request(vec!["true"]);
1344        req.execution_timeout_ms = Some(500);
1345        let token = spawn(req).expect("spawn short-lived child with deadline");
1346        let snapshot = observe_native_finalization(token);
1347        assert_eq!(snapshot.handle_state, "inert");
1348        assert_eq!(snapshot.leader_status, "completed");
1349        assert_eq!(snapshot.exited, Some(true));
1350        assert_eq!(snapshot.timed_out, Some(false));
1351    }
1352
1353    #[cfg(unix)]
1354    #[test]
1355    fn deadline_reason_survives_failed_cleanup_and_explicit_retry() {
1356        let _lock = test_guard();
1357        let token = spawn(request(vec!["sleep", "30"])).expect("spawn deadline fixture");
1358        inject::FORCE_OPERATION_ERROR.store(true, Ordering::SeqCst);
1359        assert!(terminate_with_status(token, LeaderStatus::TimedOut).is_err());
1360        let active = identity(token).expect("failed deadline retains owner");
1361        let snapshot = terminate(token).expect("explicit retry completes cleanup");
1362        let cached = wait(token, 1).expect("wait returns cached outcome");
1363        close(token).expect("close finalized fixture");
1364        assert_eq!(active.handle_state, "active");
1365        assert_eq!(active.leader_status, "running");
1366        assert_eq!(snapshot.leader_status, "timed_out");
1367        assert_eq!(cached.leader_status, "timed_out");
1368    }
1369
1370    #[cfg(windows)]
1371    #[test]
1372    fn windows_spawn_fails_before_creating_a_handle() {
1373        let _lock = test_guard();
1374        let marker = std::env::temp_dir().join(format!(
1375            "sysprims-containment-windows-{}.marker",
1376            std::process::id()
1377        ));
1378        let _ = std::fs::remove_file(&marker);
1379        let result = spawn(request(vec![
1380            "cmd",
1381            "/C",
1382            &format!("echo spawned>{}", marker.display()),
1383        ]));
1384        assert!(result.is_err());
1385        assert!(!marker.exists(), "windows rejection must not start argv");
1386    }
1387
1388    #[cfg(unix)]
1389    #[test]
1390    fn invalid_high_signal_does_not_spawn() {
1391        let _lock = test_guard();
1392        let marker = std::env::temp_dir().join(format!(
1393            "sysprims-containment-high-signal-{}.marker",
1394            std::process::id()
1395        ));
1396        let _ = std::fs::remove_file(&marker);
1397        let mut req = request(vec!["touch", marker.to_str().expect("utf8 path")]);
1398        req.signal = Some(99);
1399        assert!(spawn(req).is_err());
1400        assert!(!marker.exists(), "invalid high signal must not start argv");
1401    }
1402
1403    #[cfg(unix)]
1404    #[test]
1405    fn failed_active_close_preserves_owner_and_unreaped_is_not_inert() {
1406        let _lock = test_guard();
1407        let token = spawn(request(vec!["sleep", "30"])).expect("spawn sleep");
1408
1409        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1410        let terminated = terminate(token);
1411        assert!(terminated.is_err(), "exited:false must be retryable");
1412        let snap = identity(token).expect("owner retained after unreaped terminate");
1413        assert_eq!(snap.handle_state, "active");
1414
1415        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1416        assert!(close(token).is_err(), "unconfirmed close must fail");
1417        identity(token).expect("failed close must keep the same generation");
1418        poll(token).expect("active owner remains usable");
1419
1420        inject::FORCE_UNREAPED.store(false, Ordering::SeqCst);
1421        close(token).expect("retry close after unreaped failure");
1422        assert!(identity(token).is_err());
1423    }
1424
1425    #[cfg(unix)]
1426    #[test]
1427    fn deadline_monitor_setup_failure_returns_no_handle() {
1428        let _lock = test_guard();
1429        let before = occupied_slots();
1430        let marker = std::env::temp_dir().join(format!(
1431            "sysprims-containment-deadline-setup-{}.marker",
1432            std::process::id()
1433        ));
1434        let _ = std::fs::remove_file(&marker);
1435        inject::FORCE_DEADLINE_THREAD_FAIL.store(true, Ordering::SeqCst);
1436        let mut req = request(vec!["touch", marker.to_str().expect("utf8 path")]);
1437        req.execution_timeout_ms = Some(5_000);
1438        let result = spawn(req);
1439        inject::FORCE_DEADLINE_THREAD_FAIL.store(false, Ordering::SeqCst);
1440        assert!(
1441            result.is_err(),
1442            "deadline setup failure must not return a handle"
1443        );
1444        assert!(
1445            !marker.exists(),
1446            "deadline reservation failure must not start argv"
1447        );
1448        assert_eq!(
1449            occupied_slots(),
1450            before,
1451            "monitor-setup failure before spawn must not occupy a slot"
1452        );
1453        let token = spawn(request(vec!["true"])).expect("registry remains usable");
1454        close(token).expect("close after deadline-setup failure");
1455    }
1456
1457    #[cfg(unix)]
1458    #[test]
1459    fn failed_active_close_keeps_native_deadline_live() {
1460        let _lock = test_guard();
1461        let mut req = request(vec!["sleep", "30"]);
1462        req.execution_timeout_ms = Some(250);
1463        req.grace_timeout_ms = Some(20);
1464        let token = spawn(req).expect("spawn with deadline");
1465
1466        inject::FORCE_UNREAPED.store(true, Ordering::SeqCst);
1467        assert!(close(token).is_err(), "unconfirmed close must fail");
1468        inject::FORCE_UNREAPED.store(false, Ordering::SeqCst);
1469
1470        let snap = identity(token).expect("owner retained after failed close");
1471        assert_eq!(snap.handle_state, "active");
1472        let snap = wait(token, 5_000).expect("native deadline must still fire");
1473        assert_eq!(snap.leader_status, "timed_out");
1474        close(token).expect("close after native timeout");
1475    }
1476
1477    #[cfg(unix)]
1478    #[test]
1479    fn concurrent_spawn_and_close_does_not_deadlock() {
1480        let _lock = test_guard();
1481        let (done_tx, done_rx) = mpsc::channel();
1482        let workers = 8;
1483        let iterations = 20;
1484        for _ in 0..workers {
1485            let done_tx = done_tx.clone();
1486            thread::spawn(move || {
1487                for _ in 0..iterations {
1488                    if let Ok(token) = spawn(request(vec!["true"])) {
1489                        let _ = wait(token, 2_000);
1490                        let _ = close(token);
1491                    }
1492                }
1493                let _ = done_tx.send(());
1494            });
1495        }
1496        drop(done_tx);
1497        for _ in 0..workers {
1498            done_rx
1499                .recv_timeout(Duration::from_secs(30))
1500                .expect("spawn/close worker deadlocked or timed out");
1501        }
1502    }
1503}