Skip to main content

hermit/
ptrace_completion.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 * This source code is licensed under the BSD-style license found in LICENSE.
5 */
6
7//! Same-thread ownership of a complete ordinary-ptrace operation.
8//!
9//! A checkpoint retains the original runtime, future and every guard in that
10//! future. It is not a completed cleanup. Deliberate retention on thread exit
11//! leaves admission closed; dropping a diagnostic does not abandon the owner.
12
13use std::any::Any;
14use std::any::TypeId;
15use std::cell::Cell;
16use std::cell::RefCell;
17use std::collections::BTreeMap;
18use std::fmt;
19use std::fs::File;
20use std::future::Future;
21use std::future::poll_fn;
22use std::mem::ManuallyDrop;
23use std::os::fd::AsRawFd;
24use std::os::fd::FromRawFd;
25use std::os::fd::OwnedFd;
26use std::os::unix::fs::MetadataExt;
27use std::pin::Pin;
28use std::rc::Rc;
29use std::sync::Mutex;
30use std::sync::atomic::AtomicU64;
31use std::sync::atomic::Ordering;
32use std::task::Context;
33use std::task::Poll;
34use std::thread::ThreadId;
35use std::time::Duration;
36use std::time::Instant;
37
38use reverie_ptrace::PtraceCallbackDiagnostic;
39use reverie_ptrace::PtraceRunFailure;
40use reverie_ptrace::PtraceTerminationHandle;
41use reverie_ptrace::ToolRunOutcome;
42use serde::Deserialize;
43use serde::Serialize;
44
45use crate::Error;
46use crate::FailureKind;
47use crate::GuestTimedOut;
48
49const ATTEMPT_BUDGET: Duration = Duration::from_secs(2);
50static NEXT_KEY: AtomicU64 = AtomicU64::new(1);
51// Admission and unresolved-owner publication use this SAME lock. Operations
52// admitted before publication can finish; no later operation starts effects.
53static UNRESOLVED: Mutex<usize> = Mutex::new(0);
54
55type ErasedFuture = Pin<Box<dyn Future<Output = Result<Box<dyn Any>, Error>>>>;
56
57// Keep the pinned operation as a field, rather than an async wrapper's local,
58// when its poll unwinds. The poisoned entry is never polled again. This cannot
59// undo destructors already run by arbitrary code inside the operation's panic.
60struct EraseResult<F>(Pin<Box<F>>);
61impl<T: 'static, F: Future<Output = Result<T, Error>>> Future for EraseResult<F> {
62    type Output = Result<Box<dyn Any>, Error>;
63    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
64        self.get_mut()
65            .0
66            .as_mut()
67            .poll(cx)
68            .map(|result| result.map(|value| Box::new(value) as Box<dyn Any>))
69    }
70}
71
72thread_local! {
73    // TLS destruction must not drop a LocalSet or its backing directories.
74    // This deliberately retains resources if their original thread exits.
75    static OWNERS: RefCell<ManuallyDrop<BTreeMap<u64, Rc<Entry>>>> =
76        const { RefCell::new(ManuallyDrop::new(BTreeMap::new())) };
77}
78
79/// The phase whose original owner remains retained.
80#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
81pub enum HermitCleanupStage {
82    /// Spawn/startup has not yet returned a termination handle.
83    Startup,
84    /// The original ordinary-ptrace task, hooks or output drain are unfinished.
85    PtraceCleanup,
86    /// Physical ptrace completion returned failed global state; its natural
87    /// scheduler join or partial-recording flush remains unfinished.
88    GlobalStateCleanup,
89    /// An untouched tracer refused the ordinary completion API.
90    UnsupportedBackend,
91    /// Polling panicked. The retained operation must never be polled again.
92    Poisoned,
93}
94
95/// A completed failed ptrace run, with its original nonclone primary cause.
96/// Secondary backend/cleanup errors never determine its failure classification.
97pub struct HermitPtraceFailure {
98    failure: PtraceRunFailure,
99    cleanup: detcore::BackendFailureCleanup,
100    callbacks: Vec<PtraceCallbackDiagnostic>,
101    timeout_during_cleanup: bool,
102}
103impl HermitPtraceFailure {
104    /// The original typed failure, origin, secondary errors and captured bytes.
105    pub fn failure(&self) -> &PtraceRunFailure {
106        &self.failure
107    }
108    /// Actual failed-global-state cleanup outcomes, separate from the primary.
109    pub fn cleanup(&self) -> &detcore::BackendFailureCleanup {
110        &self.cleanup
111    }
112    /// Raw callback observations preserved by the backend.
113    pub fn callback_diagnostics(&self) -> &[PtraceCallbackDiagnostic] {
114        &self.callbacks
115    }
116    /// Whether the caller's deadline also expired during failed cleanup.
117    pub fn timeout_during_cleanup(&self) -> bool {
118        self.timeout_during_cleanup
119    }
120    pub(crate) fn kind(&self) -> FailureKind {
121        crate::error::classify_ptrace_primary(self.failure.primary())
122    }
123}
124impl fmt::Debug for HermitPtraceFailure {
125    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
126        f.debug_struct("HermitPtraceFailure")
127            .field("failure", &self.failure)
128            .field("scheduler", &self.cleanup.scheduler)
129            .field("preemption_recording", &self.cleanup.preemption_recording)
130            .field("callbacks", &self.callbacks)
131            .field("timeout_during_cleanup", &self.timeout_during_cleanup)
132            .finish()
133    }
134}
135impl fmt::Display for HermitPtraceFailure {
136    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137        self.failure.fmt(f)
138    }
139}
140impl std::error::Error for HermitPtraceFailure {
141    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
142        Some(self.failure.primary())
143    }
144}
145
146#[derive(Clone, Debug)]
147struct Snapshot {
148    kind: FailureKind,
149    primary: String,
150    origin: Option<String>,
151    stdout: Option<Vec<u8>>,
152    stderr: Option<Vec<u8>>,
153    callbacks: Vec<PtraceCallbackDiagnostic>,
154}
155impl Snapshot {
156    fn startup() -> Self {
157        Self {
158            kind: FailureKind::Error,
159            primary: "ordinary operation is not yet complete".into(),
160            origin: None,
161            stdout: None,
162            stderr: None,
163            callbacks: Vec::new(),
164        }
165    }
166    fn failure(failure: &PtraceRunFailure, callbacks: Vec<PtraceCallbackDiagnostic>) -> Self {
167        Self {
168            kind: crate::error::classify_ptrace_primary(failure.primary()),
169            primary: failure.primary().to_string(),
170            origin: Some(format!("{:?}", failure.origin())),
171            stdout: failure.captured_prefix().map(|p| p.stdout().to_vec()),
172            stderr: failure.captured_prefix().map(|p| p.stderr().to_vec()),
173            callbacks,
174        }
175    }
176}
177
178/// Sendable diagnostic identifying an operation retained on its original thread.
179///
180/// This owns no runtime or tracee capability. `resume::<T>()` must use the exact
181/// successful return type of the original public call. Dropping this value does
182/// not release the owner or reopen ordinary admission. There is no library alarm
183/// or automatic process exit after this value is returned.
184///
185/// While any operation is retained, later ordinary-ptrace calls in this process
186/// refuse admission. Already admitted calls can complete. A poisoned owner or an
187/// owner whose original thread exited cannot recover; its resources are retained
188/// and admission stays closed for the process lifetime. In-process callers must
189/// explicitly supervise this condition rather than assume a failed call cleaned
190/// up or that dropping this diagnostic released the resources.
191#[derive(Clone, Debug)]
192pub struct HermitCleanupUnconfirmed {
193    key: u64,
194    origin: OriginNumbers,
195    stage: HermitCleanupStage,
196    snapshot: Snapshot,
197}
198impl HermitCleanupUnconfirmed {
199    /// Opaque recovery key; meaningful only with the original process/thread.
200    pub fn recovery_key(&self) -> u64 {
201        self.key
202    }
203    /// The unresolved phase, not a guest exit status.
204    pub fn stage(&self) -> HermitCleanupStage {
205        self.stage
206    }
207    /// Classification of the first received failure, never a secondary cause.
208    pub fn primary_kind(&self) -> FailureKind {
209        self.snapshot.kind
210    }
211    /// A diagnostic copy. The nonclone typed cause remains with the owner.
212    pub fn primary_message(&self) -> &str {
213        &self.snapshot.primary
214    }
215    /// The original backend callback/operation, when one has been received.
216    pub fn origin(&self) -> Option<&str> {
217        self.snapshot.origin.as_deref()
218    }
219    /// Captured stdout at this checkpoint; absent for noncapturing operations.
220    pub fn stdout_prefix(&self) -> Option<&[u8]> {
221        self.snapshot.stdout.as_deref()
222    }
223    /// Captured stderr at this checkpoint; absent for noncapturing operations.
224    pub fn stderr_prefix(&self) -> Option<&[u8]> {
225        self.snapshot.stderr.as_deref()
226    }
227    /// Backend callback observations received before this checkpoint.
228    pub fn callback_diagnostics(&self) -> &[PtraceCallbackDiagnostic] {
229        &self.snapshot.callbacks
230    }
231    /// Continue the SAME operation for one cooperative two-second attempt.
232    ///
233    /// A further checkpoint is returned as another typed
234    /// `HermitCleanupUnconfirmed` error. Wrong identity/type, nested runtimes,
235    /// reentry and poisoned owners refuse without polling or discarding it.
236    /// Synchronous kernel calls and user code cannot be preempted by this budget.
237    /// If the original deadline expired before startup produced a termination
238    /// handle, recovery continues that same startup and can therefore begin its
239    /// effects after the deadline. As soon as the handle becomes available the
240    /// already-expired timeout is published; no normal guest completion is
241    /// claimed from this recovery. Do not call resume expecting cancellation of
242    /// an operation that has not yet established its owned termination handle.
243    ///
244    /// An operation a public launch function started lives on the calling
245    /// thread's tracer thread (`crate::tracer_thread`); resuming it from that
246    /// caller continues it there.
247    pub fn resume<T: Send + 'static>(&self) -> Result<T, Error> {
248        reject_nested_runtime()?;
249        let this = self.clone();
250        if let Some(resumed) =
251            crate::tracer_thread::execute_on(self.origin.thread, move || this.resume_here::<T>())
252        {
253            return resumed?;
254        }
255        self.resume_here()
256    }
257
258    fn resume_here<T: 'static>(&self) -> Result<T, Error> {
259        self.origin.check()?;
260        let entry = OWNERS
261            .try_with(|owners| owners.borrow().get(&self.key).cloned())
262            .map_err(|_| RecoveryRefusal::StorageUnavailable)?
263            .ok_or(RecoveryRefusal::UnknownKey)?;
264        if entry.result_type != TypeId::of::<T>() {
265            return Err(RecoveryRefusal::WrongResultType.into());
266        }
267        drive(entry, true)
268    }
269}
270impl fmt::Display for HermitCleanupUnconfirmed {
271    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
272        write!(
273            f,
274            "Hermit cleanup remains unconfirmed ({:?}, recovery {}): {}",
275            self.stage, self.key, self.snapshot.primary
276        )
277    }
278}
279impl std::error::Error for HermitCleanupUnconfirmed {}
280
281/// A refusal that leaves any existing operation owner untouched.
282#[derive(Debug)]
283pub enum RecoveryRefusal {
284    /// Public synchronous entry cannot nest a Tokio runtime.
285    NestedRuntime,
286    /// Recovery must run on the original thread and process/PID namespace.
287    WrongIdentity,
288    /// Current process identity could not be established.
289    Identity(std::io::Error),
290    /// The ORIGINAL process pidfd is ready or returned unexpected poll flags.
291    /// This refuses recovery; it is not proof of guest cleanup.
292    OriginalProcessUnavailable { ready: i32, events: i16 },
293    /// Thread-local storage is unavailable during thread destruction.
294    StorageUnavailable,
295    /// The key was already completed or never belonged to this thread.
296    UnknownKey,
297    /// Recovery used a different successful return type.
298    WrongResultType,
299    /// This operation is already being polled.
300    Reentrant,
301    /// A previous poll panicked; it must not be resumed.
302    Poisoned,
303    /// An unresolved owner prevents admitting another ordinary operation.
304    AdmissionClosed,
305    /// The admission mutex was poisoned; admission is conservatively refused.
306    AdmissionPoisoned,
307    /// The monotonic recovery key space is exhausted.
308    KeyExhausted,
309}
310impl fmt::Display for RecoveryRefusal {
311    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
312        write!(f, "Hermit owner recovery refused: {self:?}")
313    }
314}
315impl std::error::Error for RecoveryRefusal {}
316
317#[derive(Clone, Copy, Debug, Eq, PartialEq)]
318struct OriginNumbers {
319    thread: ThreadId,
320    pid: u32,
321    namespace_dev: u64,
322    namespace_ino: u64,
323}
324impl OriginNumbers {
325    fn current() -> Result<(Self, File), RecoveryRefusal> {
326        let file = File::open("/proc/self/ns/pid").map_err(RecoveryRefusal::Identity)?;
327        let meta = file.metadata().map_err(RecoveryRefusal::Identity)?;
328        Ok((
329            Self {
330                thread: std::thread::current().id(),
331                pid: std::process::id(),
332                namespace_dev: meta.dev(),
333                namespace_ino: meta.ino(),
334            },
335            file,
336        ))
337    }
338    fn check(&self) -> Result<(), RecoveryRefusal> {
339        let (current, _namespace) = Self::current()?;
340        if *self == current {
341            Ok(())
342        } else {
343            Err(RecoveryRefusal::WrongIdentity)
344        }
345    }
346}
347
348// A numeric PID, Rust ThreadId and namespace can all be copied across fork.
349// After the origin dies, even its PID can be reused. This descriptor keeps the
350// original kernel process identity; it is never reopened on recovery. Flags 0
351// deliberately track the process, including a surviving nonleader owner thread.
352struct OriginalProcess(OwnedFd);
353impl OriginalProcess {
354    fn capture(pid: u32) -> Result<Self, RecoveryRefusal> {
355        let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid as libc::pid_t, 0) };
356        if fd < 0 {
357            return Err(RecoveryRefusal::Identity(std::io::Error::last_os_error()));
358        }
359        Ok(Self(unsafe { OwnedFd::from_raw_fd(fd as i32) }))
360    }
361    fn check(&self) -> Result<(), RecoveryRefusal> {
362        Self::check_fd(self.0.as_raw_fd())
363    }
364    fn check_fd(fd: i32) -> Result<(), RecoveryRefusal> {
365        // poll deliberately ignores negative descriptors, so reject first.
366        if fd < 0 {
367            return Err(RecoveryRefusal::Identity(
368                std::io::Error::from_raw_os_error(libc::EBADF),
369            ));
370        }
371        let mut descriptor = libc::pollfd {
372            fd,
373            events: libc::POLLIN,
374            revents: 0,
375        };
376        let ready = unsafe { libc::poll(&mut descriptor, 1, 0) };
377        if ready < 0 {
378            return Err(RecoveryRefusal::Identity(std::io::Error::last_os_error()));
379        }
380        if ready != 0 || descriptor.revents != 0 {
381            return Err(RecoveryRefusal::OriginalProcessUnavailable {
382                ready,
383                events: descriptor.revents,
384            });
385        }
386        Ok(())
387    }
388}
389
390pub(crate) struct Control {
391    termination: RefCell<Option<PtraceTerminationHandle>>,
392    timeout: Option<(Instant, Duration)>,
393    expired: Cell<bool>,
394    received_failure: Cell<bool>,
395    cancellation_deadline: Cell<Option<Instant>>,
396    global_cleanup_deadline: Cell<Option<Instant>>,
397    stage: Cell<HermitCleanupStage>,
398    snapshot: RefCell<Snapshot>,
399    checkpoint: Cell<bool>,
400    epoch: Cell<u64>,
401}
402impl Control {
403    fn new(timeout: Option<Duration>) -> Self {
404        Self {
405            termination: RefCell::new(None),
406            timeout: timeout.map(|d| (Instant::now() + d, d)),
407            expired: Cell::new(false),
408            received_failure: Cell::new(false),
409            cancellation_deadline: Cell::new(None),
410            global_cleanup_deadline: Cell::new(None),
411            stage: Cell::new(HermitCleanupStage::Startup),
412            snapshot: RefCell::new(Snapshot::startup()),
413            checkpoint: Cell::new(false),
414            epoch: Cell::new(0),
415        }
416    }
417    fn register(&self, handle: Option<PtraceTerminationHandle>) {
418        *self.termination.borrow_mut() = handle;
419        self.stage.set(HermitCleanupStage::PtraceCleanup);
420        if self.expired.get() {
421            self.publish_timeout();
422        }
423    }
424    fn publish_timeout(&self) {
425        if let (Some(handle), Some((_, limit))) = (self.termination.borrow().as_ref(), self.timeout)
426        {
427            // Reverie preserves an earlier Tool failure as primary.
428            handle.terminate(reverie::Error::Tool(Error::new(GuestTimedOut { limit })));
429        }
430    }
431    /// Mark the run timed out if `now` is at or past its deadline. The caller
432    /// passes the same `now` it then compares deadlines against, so a deadline
433    /// can never be seen as passed while the timeout is still unexpired.
434    fn expire_if_due(&self, now: Instant) {
435        if !self.expired.get() && self.timeout.is_some_and(|(at, _)| now >= at) {
436            self.expired.set(true);
437            if !self.received_failure.get() {
438                let mut snapshot = self.snapshot.borrow_mut();
439                snapshot.kind = FailureKind::RunTimeout;
440                snapshot.primary = GuestTimedOut {
441                    limit: self.timeout.unwrap().1,
442                }
443                .to_string();
444            }
445            self.publish_timeout();
446            self.cancellation_deadline
447                .set(Some(Instant::now() + ATTEMPT_BUDGET));
448        }
449    }
450    async fn yield_checkpoint(&self) {
451        let epoch = self.epoch.get();
452        self.checkpoint.set(true);
453        poll_fn(|_| {
454            if self.epoch.get() != epoch {
455                Poll::Ready(())
456            } else {
457                Poll::Pending
458            }
459        })
460        .await;
461    }
462    fn global_failure(&self, failure: &PtraceRunFailure, callbacks: Vec<PtraceCallbackDiagnostic>) {
463        self.termination.borrow_mut().take();
464        self.received_failure.set(true);
465        *self.snapshot.borrow_mut() = Snapshot::failure(failure, callbacks);
466        self.stage.set(HermitCleanupStage::GlobalStateCleanup);
467        self.global_cleanup_deadline
468            .set(Some(Instant::now() + ATTEMPT_BUDGET));
469    }
470}
471
472struct Driver {
473    runtime: tokio::runtime::Runtime,
474    future: ErasedFuture,
475    control: Rc<Control>,
476}
477struct Entry {
478    key: u64,
479    origin: OriginNumbers,
480    _namespace: File,
481    original_process: OriginalProcess,
482    result_type: TypeId,
483    driver: RefCell<ManuallyDrop<Driver>>,
484    in_flight: Cell<bool>,
485    poisoned: Cell<bool>,
486    quarantined: Cell<bool>,
487}
488impl Entry {
489    fn check_origin(&self) -> Result<(), RecoveryRefusal> {
490        self.origin.check()?;
491        self.original_process.check()
492    }
493}
494
495struct PollGuard<'a>(&'a Cell<bool>);
496impl Drop for PollGuard<'_> {
497    fn drop(&mut self) {
498        self.0.set(false);
499    }
500}
501fn reject_nested_runtime() -> Result<(), RecoveryRefusal> {
502    if tokio::runtime::Handle::try_current().is_ok() {
503        Err(RecoveryRefusal::NestedRuntime)
504    } else {
505        Ok(())
506    }
507}
508
509/// [`run`] on the calling thread's tracer thread (`crate::tracer_thread`), with
510/// its explicit stack, so that a library launch needs no particular stack from
511/// its caller. A nested runtime on the calling thread is refused here, before
512/// anything moves to the tracer thread.
513pub(crate) fn run_on_tracer_thread<T, F, Make>(
514    timeout: Option<Duration>,
515    make: Make,
516) -> Result<T, Error>
517where
518    T: Send + 'static,
519    F: Future<Output = Result<T, Error>> + 'static,
520    Make: FnOnce(Rc<Control>) -> F + Send + 'static,
521{
522    reject_nested_runtime()?;
523    crate::tracer_thread::execute(move || run(timeout, make))?
524}
525
526/// Installs the whole operation before its first poll. `make` must only construct
527/// a future; every ordinary effect and all owned guards belong inside that future.
528pub(crate) fn run<T, F, Make>(timeout: Option<Duration>, make: Make) -> Result<T, Error>
529where
530    T: 'static,
531    F: Future<Output = Result<T, Error>> + 'static,
532    Make: FnOnce(Rc<Control>) -> F,
533{
534    reject_nested_runtime()?;
535    let (origin, namespace) = OriginNumbers::current()?;
536    let original_process = OriginalProcess::capture(origin.pid)?;
537    let key = NEXT_KEY
538        .try_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_add(1))
539        .map_err(|_| RecoveryRefusal::KeyExhausted)?;
540    let control = Rc::new(Control::new(timeout));
541    let runtime = tokio::runtime::Builder::new_current_thread()
542        .enable_all()
543        .build()?;
544    let admission = UNRESOLVED
545        .lock()
546        .map_err(|_| RecoveryRefusal::AdmissionPoisoned)?;
547    if *admission != 0 {
548        return Err(RecoveryRefusal::AdmissionClosed.into());
549    }
550    // This is the admission linearization, before make or any first poll.
551    drop(admission);
552    let future = make(control.clone());
553    let future: ErasedFuture = Box::pin(EraseResult(Box::pin(future)));
554    let entry = Rc::new(Entry {
555        key,
556        origin,
557        _namespace: namespace,
558        original_process,
559        result_type: TypeId::of::<T>(),
560        driver: RefCell::new(ManuallyDrop::new(Driver {
561            runtime,
562            future,
563            control,
564        })),
565        in_flight: Cell::new(false),
566        poisoned: Cell::new(false),
567        quarantined: Cell::new(false),
568    });
569    OWNERS
570        .try_with(|owners| {
571            owners.borrow_mut().insert(key, entry.clone());
572        })
573        .map_err(|_| RecoveryRefusal::StorageUnavailable)?;
574    drive(entry, false)
575}
576
577fn drive<T: 'static>(entry: Rc<Entry>, recovery: bool) -> Result<T, Error> {
578    entry.check_origin()?;
579    if entry.poisoned.get() {
580        return Err(RecoveryRefusal::Poisoned.into());
581    }
582    if entry.in_flight.replace(true) {
583        return Err(RecoveryRefusal::Reentrant.into());
584    }
585    let _poll_guard = PollGuard(&entry.in_flight);
586    let attempt_deadline = recovery.then(|| Instant::now() + ATTEMPT_BUDGET);
587    let polled = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
588        let mut stored = entry.driver.borrow_mut();
589        let Driver {
590            runtime,
591            future,
592            control,
593        } = &mut **stored;
594        if recovery {
595            control.checkpoint.set(false);
596            control.epoch.set(control.epoch.get().wrapping_add(1));
597        }
598        runtime.block_on(async {
599            let mut timer: Option<Pin<Box<tokio::time::Sleep>>> = None;
600            poll_fn(|cx| {
601                control.expire_if_due(Instant::now());
602                if !recovery
603                    && control.expired.get()
604                    && control.stage.get() == HermitCleanupStage::Startup
605                {
606                    return Poll::Ready(None);
607                }
608                let polled = future.as_mut().poll(cx);
609                // A synchronous poll can cross the absolute deadline. Observe
610                // that fact on its return without claiming preemption. This one
611                // reading of the clock also decides the deadline check below:
612                // with a second reading, the timeout could pass between the two,
613                // and the check would end the poll with the timeout unexpired,
614                // reporting HermitCleanupUnconfirmed with the startup snapshot
615                // instead of a run timeout (dh-4s1xh).
616                let now = Instant::now();
617                control.expire_if_due(now);
618                if let Poll::Ready(result) = polled {
619                    let result = match (result, control.timeout) {
620                        (Ok(_), Some((_, limit))) if control.expired.get() => {
621                            Err(Error::new(GuestTimedOut { limit }))
622                        }
623                        (result, _) => result,
624                    };
625                    return Poll::Ready(Some(result));
626                }
627                if control.checkpoint.get() {
628                    return Poll::Ready(None);
629                }
630                let deadline = if recovery {
631                    attempt_deadline
632                } else {
633                    [
634                        control
635                            .timeout
636                            .filter(|_| !control.expired.get())
637                            .map(|(at, _)| at),
638                        control.cancellation_deadline.get(),
639                        control.global_cleanup_deadline.get(),
640                    ]
641                    .into_iter()
642                    .flatten()
643                    .min()
644                };
645                if let Some(deadline) = deadline {
646                    if now >= deadline {
647                        return Poll::Ready(None);
648                    }
649                    let at = tokio::time::Instant::from_std(deadline);
650                    match timer.as_mut() {
651                        Some(timer) => timer.as_mut().reset(at),
652                        None => timer = Some(Box::pin(tokio::time::sleep_until(at))),
653                    }
654                    if timer.as_mut().unwrap().as_mut().poll(cx).is_ready() {
655                        cx.waker().wake_by_ref();
656                    }
657                }
658                Poll::Pending
659            })
660            .await
661        })
662    }));
663    match polled {
664        Ok(Some(result)) => {
665            OWNERS.with(|owners| {
666                owners.borrow_mut().remove(&entry.key);
667            });
668            if entry.quarantined.replace(false) {
669                let mut unresolved = UNRESOLVED
670                    .lock()
671                    .unwrap_or_else(|poison| poison.into_inner());
672                *unresolved -= 1;
673            }
674            // Runtime teardown happens only after the whole operation returned,
675            // outside block_on. No unresolved future is dropped here.
676            let driver = unsafe { ManuallyDrop::take(&mut *entry.driver.borrow_mut()) };
677            drop(driver);
678            result?
679                .downcast::<T>()
680                .map(|value| *value)
681                .map_err(|_| RecoveryRefusal::WrongResultType.into())
682        }
683        outcome => {
684            if !entry.quarantined.replace(true) {
685                let mut unresolved = UNRESOLVED
686                    .lock()
687                    .unwrap_or_else(|poison| poison.into_inner());
688                *unresolved = unresolved.saturating_add(1);
689            }
690            if outcome.is_err() {
691                entry.poisoned.set(true);
692            }
693            let stored = entry.driver.borrow();
694            let control = &stored.control;
695            if entry.poisoned.get() {
696                control.stage.set(HermitCleanupStage::Poisoned);
697                if !control.received_failure.get() && !control.expired.get() {
698                    let mut snapshot = control.snapshot.borrow_mut();
699                    snapshot.kind = FailureKind::Panic;
700                    snapshot.primary =
701                        "ordinary operation panicked while polling; owner is poisoned".into();
702                }
703            }
704            Err(Error::new(HermitCleanupUnconfirmed {
705                key: entry.key,
706                origin: entry.origin,
707                stage: control.stage.get(),
708                snapshot: control.snapshot.borrow().clone(),
709            }))
710        }
711    }
712}
713
714pub(crate) async fn wait(
715    tracer: reverie_ptrace::Tracer<detcore::GlobalState>,
716    control: Rc<Control>,
717) -> Result<(reverie::ExitStatus, detcore::GlobalState), Error> {
718    control.register(tracer.termination_handle());
719    consume(tracer.wait_completion().await, control).await
720}
721pub(crate) async fn wait_with_output(
722    tracer: reverie_ptrace::Tracer<detcore::GlobalState>,
723    control: Rc<Control>,
724) -> Result<(reverie::process::Output, detcore::GlobalState), Error> {
725    control.register(tracer.termination_handle());
726    consume(tracer.wait_with_output_completion().await, control).await
727}
728async fn consume<R>(
729    mut outcome: ToolRunOutcome<detcore::GlobalState, R>,
730    control: Rc<Control>,
731) -> Result<(R, detcore::GlobalState), Error> {
732    loop {
733        match outcome {
734            ToolRunOutcome::Complete(completion) => {
735                let callbacks = completion.callback_diagnostics().to_vec();
736                control.termination.borrow_mut().take();
737                return match completion.result {
738                    Ok(result) => Ok((result, completion.global_state)),
739                    Err(failure) => {
740                        control.global_failure(&failure, callbacks.clone());
741                        let cleanup = completion
742                            .global_state
743                            .clean_up_after_backend_failure()
744                            .await;
745                        Err(HermitPtraceFailure {
746                            failure,
747                            cleanup,
748                            callbacks,
749                            timeout_during_cleanup: control.expired.get(),
750                        }
751                        .into())
752                    }
753                };
754            }
755            ToolRunOutcome::CleanupPending(pending) => {
756                control.received_failure.set(true);
757                *control.snapshot.borrow_mut() =
758                    Snapshot::failure(pending.failure(), pending.callback_diagnostics());
759                control.stage.set(HermitCleanupStage::PtraceCleanup);
760                control.yield_checkpoint().await;
761                outcome = pending.resume_cleanup().await;
762            }
763            ToolRunOutcome::UnsupportedBackend(tracer) => {
764                control.stage.set(HermitCleanupStage::UnsupportedBackend);
765                control.snapshot.borrow_mut().primary =
766                    "configured tracer does not support ordinary owned completion".into();
767                // The untouched tracer remains in this async frame forever;
768                // repeated recovery refuses at a checkpoint, never legacy wait.
769                loop {
770                    std::hint::black_box(&tracer);
771                    control.yield_checkpoint().await;
772                }
773            }
774        }
775    }
776}
777
778#[cfg(test)]
779#[path = "ptrace_completion_tests.rs"]
780mod tests;