Skip to main content

khive_runtime/
resource_admission.rs

1//! Class admission and isolated execution for explicitly attributed background jobs.
2//! Interactive dispatch does not enter this module. A job's lease, not its awaiter,
3//! owns admission and tracking through the final actual blocking-child exit.
4
5use std::fmt;
6use std::future::Future;
7use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
8use std::sync::{Arc, Mutex, MutexGuard};
9use std::time::{Duration, Instant};
10use tokio::runtime::Handle;
11use tokio::sync::{watch, OwnedSemaphorePermit, Semaphore};
12use tokio::task::JoinHandle;
13
14mod priority;
15#[cfg(test)]
16mod tests;
17
18#[derive(Clone, Copy, Debug, Eq, PartialEq)]
19pub enum WorkClass {
20    Warm,
21    Maintenance,
22}
23
24#[derive(Clone, Copy, Debug, Eq, PartialEq)]
25pub struct WorkClassLimits {
26    warm: usize,
27    maintenance: usize,
28}
29impl WorkClassLimits {
30    pub fn validate(warm: usize, maintenance: usize) -> Result<Self, ResourceError> {
31        for (class, value) in [
32            (WorkClass::Warm, warm),
33            (WorkClass::Maintenance, maintenance),
34        ] {
35            if value == 0 || value > Semaphore::MAX_PERMITS {
36                return Err(ResourceError::InvalidLimit {
37                    class,
38                    value,
39                    max: Semaphore::MAX_PERMITS,
40                });
41            }
42        }
43        Ok(Self { warm, maintenance })
44    }
45    pub fn warm(self) -> usize {
46        self.warm
47    }
48    pub fn maintenance(self) -> usize {
49        self.maintenance
50    }
51}
52
53#[derive(Clone, Copy, Debug)]
54pub struct CleanupPolicy {
55    grace: Duration,
56}
57impl CleanupPolicy {
58    pub fn new(grace: Duration) -> Self {
59        Self { grace }
60    }
61    pub fn grace(self) -> Duration {
62        self.grace
63    }
64    pub fn deadline_from(self, now: Instant) -> Result<Instant, ResourceError> {
65        now.checked_add(self.grace)
66            .ok_or(ResourceError::DeadlineOverflow)
67    }
68}
69
70#[derive(Clone, Debug)]
71pub enum PriorityError {
72    Unsupported {
73        target: &'static str,
74    },
75    Os {
76        operation: &'static str,
77        code: Option<i32>,
78    },
79    Unexpected {
80        operation: &'static str,
81        value: i32,
82    },
83}
84#[derive(Clone, Debug)]
85pub enum ResourceError {
86    InvalidLimit {
87        class: WorkClass,
88        value: usize,
89        max: usize,
90    },
91    DeadlineOverflow,
92    Closed,
93    WrongDomain,
94    Priority(PriorityError),
95    ThreadStart(String),
96    RuntimeStart(String),
97    CoordinatorGone,
98    JoinFailed(String),
99}
100#[derive(Debug)]
101pub enum BackgroundJoinError {
102    Cancelled,
103    Panicked,
104    Other(String),
105}
106macro_rules! display_error {
107    ($($ty:ty),+ $(,)?) => {$ (
108        impl fmt::Display for $ty {
109            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self:?}") }
110        }
111        impl std::error::Error for $ty {}
112    )+ };
113}
114display_error!(PriorityError, ResourceError, BackgroundJoinError);
115
116#[derive(Debug)]
117pub enum BodyOutcome<T> {
118    Finished(T),
119    Cancelled,
120}
121#[derive(Debug)]
122pub enum JobOutcome<T> {
123    Completed(T),
124    Cancelled,
125    Refused(PriorityError),
126    FinalizationTimedOut,
127}
128#[derive(Clone, Copy, Debug, Eq, PartialEq)]
129pub enum FinalizationOutcome {
130    AttemptReturned,
131    AttemptTimedOut,
132    NoAttemptBeforeDeadline,
133    NoTelemetryToken,
134    AlreadyFinalized,
135    DeadlineOverflow,
136}
137#[derive(Clone, Debug)]
138pub struct ShutdownReport {
139    pub workers_exited: bool,
140    pub coordinator_joined: bool,
141    pub outstanding_jobs: usize,
142    pub incomplete_finalizations: usize,
143}
144
145fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
146    mutex
147        .lock()
148        .unwrap_or_else(std::sync::PoisonError::into_inner)
149}
150fn shorten(slot: &mut Option<Instant>, deadline: Instant) {
151    *slot = Some(slot.map_or(deadline, |old| old.min(deadline)));
152}
153struct StartGate {
154    closed: bool,
155    deadline: Option<Instant>,
156}
157struct JoinState {
158    coordinator: Option<std::thread::JoinHandle<()>>,
159    joining: bool,
160    joined: Option<Result<(), ResourceError>>,
161}
162struct Domain {
163    limits: WorkClassLimits,
164    cleanup: CleanupPolicy,
165    warm: Arc<Semaphore>,
166    maintenance: Arc<Semaphore>,
167    gate: Mutex<StartGate>,
168    changed: watch::Sender<()>,
169    supervisors: AtomicUsize,
170    jobs: AtomicUsize,
171    incomplete: AtomicUsize,
172    worker_exit: Mutex<Option<Result<(), ResourceError>>>,
173    join: Mutex<JoinState>,
174    posture_failure: Mutex<Option<PriorityError>>,
175    // Instance-local source-test seams at the real enqueue/publication boundary.
176    #[cfg(test)]
177    after_blocking_enqueue: Mutex<Option<Box<dyn FnOnce() + Send>>>,
178    #[cfg(test)]
179    blocking_dequeued: Mutex<Option<std::sync::mpsc::Sender<()>>>,
180}
181impl Domain {
182    fn new(limits: WorkClassLimits, cleanup: CleanupPolicy) -> Arc<Self> {
183        let (changed, _) = watch::channel(());
184        Arc::new(Self {
185            limits,
186            cleanup,
187            warm: Arc::new(Semaphore::new(limits.warm)),
188            maintenance: Arc::new(Semaphore::new(limits.maintenance)),
189            gate: Mutex::new(StartGate {
190                closed: false,
191                deadline: None,
192            }),
193            changed,
194            supervisors: AtomicUsize::new(0),
195            jobs: AtomicUsize::new(0),
196            incomplete: AtomicUsize::new(0),
197            worker_exit: Mutex::new(None),
198            join: Mutex::new(JoinState {
199                coordinator: None,
200                joining: false,
201                joined: None,
202            }),
203            posture_failure: Mutex::new(None),
204            #[cfg(test)]
205            after_blocking_enqueue: Mutex::new(None),
206            #[cfg(test)]
207            blocking_dequeued: Mutex::new(None),
208        })
209    }
210    fn notify(&self) {
211        let _ = self.changed.send(());
212    }
213    fn close(&self, deadline: Instant) {
214        {
215            let mut gate = lock(&self.gate);
216            gate.closed = true;
217            shorten(&mut gate.deadline, deadline);
218            self.warm.close();
219            self.maintenance.close();
220        }
221        self.notify();
222    }
223    fn close_with_policy(&self) -> Result<(), ResourceError> {
224        let now = Instant::now();
225        match self.cleanup.deadline_from(now) {
226            Ok(deadline) => {
227                self.close(deadline);
228                Ok(())
229            }
230            Err(error) => {
231                self.close(now);
232                Err(error)
233            }
234        }
235    }
236    fn closed_error(&self) -> ResourceError {
237        lock(&self.posture_failure)
238            .clone()
239            .map_or(ResourceError::Closed, ResourceError::Priority)
240    }
241    fn posture_failed(&self, error: PriorityError) {
242        tracing::error!(error = %error, "isolated background worker priority failed; closing domain");
243        let mut failure = lock(&self.posture_failure);
244        if failure.is_none() {
245            *failure = Some(error);
246        }
247        drop(failure);
248        let _ = self.close_with_policy();
249    }
250    fn report(&self) -> ShutdownReport {
251        ShutdownReport {
252            workers_exited: lock(&self.worker_exit).is_some(),
253            coordinator_joined: lock(&self.join).joined.as_ref().is_some_and(Result::is_ok),
254            outstanding_jobs: self.jobs.load(Ordering::SeqCst),
255            incomplete_finalizations: self.incomplete.load(Ordering::SeqCst),
256        }
257    }
258    async fn drive_until_closed(&self) {
259        let mut changed = self.changed.subscribe();
260        loop {
261            let (closed, deadline) = {
262                let gate = lock(&self.gate);
263                (gate.closed, gate.deadline)
264            };
265            if closed {
266                if self.supervisors.load(Ordering::SeqCst) == 0 {
267                    return;
268                }
269                let remaining = deadline
270                    .unwrap_or_else(Instant::now)
271                    .saturating_duration_since(Instant::now());
272                if remaining.is_zero() {
273                    return;
274                }
275                tokio::select! {
276                    _ = changed.changed() => {},
277                    _ = tokio::time::sleep(remaining) => {},
278                }
279            } else if changed.changed().await.is_err() {
280                return;
281            }
282        }
283    }
284}
285
286struct ExternalOwner {
287    domain: Arc<Domain>,
288    executor: Handle,
289}
290impl Drop for ExternalOwner {
291    fn drop(&mut self) {
292        let _ = self.domain.close_with_policy();
293    }
294}
295#[derive(Clone)]
296pub struct BackgroundRoot {
297    owner: Arc<ExternalOwner>,
298}
299pub type BackgroundDomainOwner = BackgroundRoot;
300#[derive(Clone)]
301pub struct BackgroundDomainHandle {
302    domain: Arc<Domain>,
303    executor: Handle,
304}
305#[derive(Clone)]
306pub struct BackgroundDomainControl {
307    domain: Arc<Domain>,
308}
309
310impl BackgroundRoot {
311    pub fn start(limits: WorkClassLimits, cleanup: CleanupPolicy) -> Result<Self, ResourceError> {
312        Self::start_inner(limits, cleanup, None)
313    }
314    fn start_inner(
315        limits: WorkClassLimits,
316        cleanup: CleanupPolicy,
317        blocking_threads: Option<usize>,
318    ) -> Result<Self, ResourceError> {
319        // Refuse a future deadline that cannot be represented before allocating workers.
320        cleanup.deadline_from(Instant::now())?;
321        let domain = Domain::new(limits, cleanup);
322        let worker_domain = domain.clone();
323        let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1);
324        let coordinator = std::thread::Builder::new()
325            .name("khive-background".into())
326            .spawn(move || {
327                let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
328                    if let Err(error) = priority::lower_current_thread() {
329                        let error = ResourceError::Priority(error);
330                        let _ = ready_tx.send(Err(error.clone()));
331                        return Err(error);
332                    }
333                    let hook_domain = worker_domain.clone();
334                    let mut builder = tokio::runtime::Builder::new_current_thread();
335                    if let Some(count) = blocking_threads {
336                        builder.max_blocking_threads(count);
337                    }
338                    let runtime = match builder
339                        .enable_all()
340                        .on_thread_start(move || {
341                            if let Err(error) = priority::lower_current_thread() {
342                                hook_domain.posture_failed(error);
343                            }
344                        })
345                        .build()
346                    {
347                        Ok(runtime) => runtime,
348                        Err(error) => {
349                            let error = ResourceError::RuntimeStart(error.to_string());
350                            let _ = ready_tx.send(Err(error.clone()));
351                            return Err(error);
352                        }
353                    };
354                    if ready_tx.send(Ok(runtime.handle().clone())).is_err() {
355                        worker_domain.close(Instant::now());
356                    }
357                    runtime.block_on(worker_domain.drive_until_closed());
358                    // Deliberately on this standard thread. Async tasks are dropped, but
359                    // Runtime destruction still joins actual started blocking work.
360                    drop(runtime);
361                    Ok(())
362                }))
363                .unwrap_or_else(|_| Err(ResourceError::JoinFailed("coordinator panicked".into())));
364                *lock(&worker_domain.worker_exit) = Some(result);
365                worker_domain.notify();
366            })
367            .map_err(|error| ResourceError::ThreadStart(error.to_string()))?;
368        lock(&domain.join).coordinator = Some(coordinator);
369        let executor = ready_rx
370            .recv()
371            .map_err(|_| ResourceError::CoordinatorGone)??;
372        Ok(Self {
373            owner: Arc::new(ExternalOwner { domain, executor }),
374        })
375    }
376    pub fn handle(&self) -> BackgroundDomainHandle {
377        BackgroundDomainHandle {
378            domain: self.owner.domain.clone(),
379            executor: self.owner.executor.clone(),
380        }
381    }
382    pub fn control(&self) -> BackgroundDomainControl {
383        self.handle().control()
384    }
385    pub fn limits(&self) -> WorkClassLimits {
386        self.owner.domain.limits
387    }
388    pub fn cleanup_policy(&self) -> CleanupPolicy {
389        self.owner.domain.cleanup
390    }
391    pub fn same_domain(&self, other: &Self) -> bool {
392        Arc::ptr_eq(&self.owner.domain, &other.owner.domain)
393    }
394}
395
396impl BackgroundDomainControl {
397    /// Observe domain closure without retaining its external root owner.
398    /// Subscribe before the state read so a close between the read and await
399    /// cannot be lost. This wait never mutates admission or exposes a token.
400    pub async fn closed(&self) {
401        let mut changed = self.domain.changed.subscribe();
402        while !self.is_closed() {
403            // self retains the sender, so the channel cannot close while this
404            // wait is live. A version change triggers another state read.
405            let _ = changed.changed().await;
406        }
407    }
408    pub fn close(&self, deadline: Instant) {
409        self.domain.close(deadline);
410    }
411    pub fn close_with_policy(&self) -> Result<(), ResourceError> {
412        self.domain.close_with_policy()
413    }
414    pub fn is_closed(&self) -> bool {
415        lock(&self.domain.gate).closed
416    }
417    fn ensure_join_started(&self) -> Result<(), ResourceError> {
418        let mut join = lock(&self.domain.join);
419        if join.joining || join.joined.is_some() {
420            return Ok(());
421        }
422        let Some(coordinator) = join.coordinator.take() else {
423            return Err(ResourceError::CoordinatorGone);
424        };
425        // The envelope returns ownership on thread-creation failure. A detached
426        // reaper, not the requesting async future, owns completion publication.
427        let envelope = Arc::new(Mutex::new(Some(coordinator)));
428        let envelope_worker = envelope.clone();
429        let domain = self.domain.clone();
430        join.joining = true;
431        let spawned = std::thread::Builder::new()
432            .name("khive-background-join".into())
433            .spawn(move || {
434                let coordinator = lock(&envelope_worker)
435                    .take()
436                    .expect("single reaper owns coordinator");
437                let result = coordinator
438                    .join()
439                    .map_err(|_| ResourceError::JoinFailed("coordinator panicked".into()))
440                    .and_then(|()| {
441                        lock(&domain.worker_exit)
442                            .clone()
443                            .unwrap_or(Err(ResourceError::CoordinatorGone))
444                    });
445                let mut join = lock(&domain.join);
446                join.joined = Some(result);
447                join.joining = false;
448                drop(join);
449                domain.notify();
450            });
451        match spawned {
452            Ok(reaper) => {
453                drop(reaper);
454                Ok(())
455            }
456            Err(error) => {
457                join.coordinator = lock(&envelope).take();
458                join.joining = false;
459                Err(ResourceError::ThreadStart(error.to_string()))
460            }
461        }
462    }
463    /// Observe actual infrastructure exit using only this caller's budget.
464    /// May start the independently owned reaper, but never closes admission,
465    /// requests cancellation, or changes any domain/job cleanup deadline.
466    pub async fn observe_exit_until(
467        &self,
468        deadline: Instant,
469    ) -> Result<ShutdownReport, ResourceError> {
470        self.ensure_join_started()?;
471        let mut changed = self.domain.changed.subscribe();
472        loop {
473            let joined = { lock(&self.domain.join).joined.clone() };
474            if let Some(result) = joined {
475                result?;
476                return Ok(self.domain.report());
477            }
478            let remaining = deadline.saturating_duration_since(Instant::now());
479            if remaining.is_zero() {
480                return Ok(self.domain.report());
481            }
482            tokio::select! {
483                _ = changed.changed() => {},
484                _ = tokio::time::sleep(remaining) => {},
485            }
486        }
487    }
488    pub async fn shutdown(&self, deadline: Instant) -> Result<ShutdownReport, ResourceError> {
489        self.close(deadline);
490        self.ensure_join_started()?;
491        let mut changed = self.domain.changed.subscribe();
492        loop {
493            let joined = { lock(&self.domain.join).joined.clone() };
494            if let Some(result) = joined {
495                result?;
496                return Ok(self.domain.report());
497            }
498            // Every observer shares the first/shortest cleanup deadline. A
499            // notification must re-read it, never restart the caller's budget.
500            let effective = lock(&self.domain.gate).deadline.unwrap_or(deadline);
501            let remaining = effective.saturating_duration_since(Instant::now());
502            if remaining.is_zero() {
503                return Ok(self.domain.report());
504            }
505            tokio::select! {
506                _ = changed.changed() => {},
507                _ = tokio::time::sleep(remaining) => {},
508            }
509        }
510    }
511}
512
513#[derive(Clone)]
514pub struct TrackedJob {
515    _guard: Arc<crate::daemon::BackgroundTaskGuard>,
516}
517impl TrackedJob {
518    pub fn register(name: &'static str) -> Self {
519        Self {
520            _guard: Arc::new(crate::daemon::register_background_task(name)),
521        }
522    }
523}
524struct JobCancellation {
525    domain: Arc<Domain>,
526    requested: AtomicBool,
527    deadline: Mutex<Option<Instant>>,
528    deadline_overflow: AtomicBool,
529    // 0 unclaimed, 1 in progress, 2 returned/skipped, 3 incomplete.
530    finalization: AtomicU8,
531}
532impl JobCancellation {
533    fn request(&self) {
534        let gate = lock(&self.domain.gate);
535        if !self.requested.swap(true, Ordering::SeqCst) {
536            let now = Instant::now();
537            let deadline = self.domain.cleanup.deadline_from(now).unwrap_or_else(|_| {
538                self.deadline_overflow.store(true, Ordering::SeqCst);
539                now
540            });
541            shorten(&mut lock(&self.deadline), deadline);
542        }
543        drop(gate);
544        self.domain.notify();
545    }
546    fn is_cancelled(&self) -> bool {
547        lock(&self.domain.gate).closed || self.requested.load(Ordering::SeqCst)
548    }
549    fn effective_deadline(&self) -> Result<Instant, ResourceError> {
550        let gate = lock(&self.domain.gate);
551        let mut own = lock(&self.deadline);
552        if own.is_none() {
553            *own = Some(self.domain.cleanup.deadline_from(Instant::now())?);
554        }
555        Ok(gate
556            .deadline
557            .map_or(own.expect("deadline initialized"), |d| {
558                d.min(own.expect("deadline initialized"))
559            }))
560    }
561}
562struct Lease {
563    _permit: OwnedSemaphorePermit,
564    _tracking: TrackedJob,
565    cancel: Arc<JobCancellation>,
566    executor: Handle,
567    class: WorkClass,
568}
569impl Drop for Lease {
570    fn drop(&mut self) {
571        self.cancel.domain.jobs.fetch_sub(1, Ordering::SeqCst);
572        self.cancel.domain.notify();
573    }
574}
575#[derive(Clone)]
576pub struct AdmittedJob {
577    lease: Arc<Lease>,
578}
579
580struct SupervisorGuard {
581    cancel: Arc<JobCancellation>,
582    finished: bool,
583}
584impl Drop for SupervisorGuard {
585    fn drop(&mut self) {
586        let phase = self.cancel.finalization.load(Ordering::SeqCst);
587        if (!self.finished && phase < 2) || phase == 1 {
588            self.cancel.domain.incomplete.fetch_add(1, Ordering::SeqCst);
589        }
590        self.cancel
591            .domain
592            .supervisors
593            .fetch_sub(1, Ordering::SeqCst);
594        self.cancel.domain.notify();
595    }
596}
597fn start_decision(cancel: &JobCancellation) -> Result<(), JobOutcome<()>> {
598    let gate = lock(&cancel.domain.gate);
599    start_decision_locked(cancel, &gate)
600}
601fn start_decision_locked(cancel: &JobCancellation, gate: &StartGate) -> Result<(), JobOutcome<()>> {
602    if let Some(error) = lock(&cancel.domain.posture_failure).clone() {
603        return Err(JobOutcome::Refused(error));
604    }
605    if gate.closed || cancel.requested.load(Ordering::SeqCst) {
606        return Err(JobOutcome::Cancelled);
607    }
608    if let Err(error) = priority::check_current_thread() {
609        return Err(JobOutcome::Refused(error));
610    }
611    // Returning Ok linearizes Started before releasing the gate. A later cancel
612    // cannot revoke this body; its actual closure owns the lease until exit.
613    Ok(())
614}
615fn refusal<T>(outcome: JobOutcome<()>) -> JobOutcome<T> {
616    match outcome {
617        JobOutcome::Cancelled => JobOutcome::Cancelled,
618        JobOutcome::Refused(error) => JobOutcome::Refused(error),
619        _ => unreachable!("start decision only refuses cancellation or posture"),
620    }
621}
622impl BackgroundDomainHandle {
623    /// Read-only domain-close observation, including final root-owner drop.
624    pub async fn closed(&self) {
625        self.control().closed().await;
626    }
627    pub fn limits(&self) -> WorkClassLimits {
628        self.domain.limits
629    }
630    pub fn control(&self) -> BackgroundDomainControl {
631        BackgroundDomainControl {
632            domain: self.domain.clone(),
633        }
634    }
635    pub async fn admit(
636        &self,
637        class: WorkClass,
638        tracking: &TrackedJob,
639    ) -> Result<AdmittedJob, ResourceError> {
640        let semaphore = match class {
641            WorkClass::Warm => self.domain.warm.clone(),
642            WorkClass::Maintenance => self.domain.maintenance.clone(),
643        };
644        let permit = semaphore
645            .acquire_owned()
646            .await
647            .map_err(|_| self.domain.closed_error())?;
648        let gate = lock(&self.domain.gate);
649        if gate.closed {
650            return Err(self.domain.closed_error());
651        }
652        self.domain.jobs.fetch_add(1, Ordering::SeqCst);
653        let cancel = Arc::new(JobCancellation {
654            domain: self.domain.clone(),
655            requested: AtomicBool::new(false),
656            deadline: Mutex::new(None),
657            deadline_overflow: AtomicBool::new(false),
658            finalization: AtomicU8::new(0),
659        });
660        Ok(AdmittedJob {
661            lease: Arc::new(Lease {
662                _permit: permit,
663                _tracking: tracking.clone(),
664                cancel,
665                executor: self.executor.clone(),
666                class,
667            }),
668        })
669    }
670    pub fn spawn_supervised<F, Fut, T>(
671        &self,
672        job: AdmittedJob,
673        make: F,
674    ) -> Result<BackgroundJob<T>, ResourceError>
675    where
676        F: FnOnce(AdmittedJob) -> Fut + Send + 'static,
677        Fut: Future<Output = JobOutcome<T>> + Send + 'static,
678        T: Send + 'static,
679    {
680        if !Arc::ptr_eq(&self.domain, &job.lease.cancel.domain) {
681            return Err(ResourceError::WrongDomain);
682        }
683        let gate = lock(&self.domain.gate);
684        if gate.closed {
685            return Err(self.domain.closed_error());
686        }
687        let cancel = job.lease.cancel.clone();
688        self.domain.supervisors.fetch_add(1, Ordering::SeqCst);
689        let guard = SupervisorGuard {
690            cancel: cancel.clone(),
691            finished: false,
692        };
693        let task = self.executor.spawn(async move {
694            let mut guard = guard;
695            let execution_hold = job.clone();
696            let outcome = match start_decision(&job.lease.cancel) {
697                Ok(()) => make(job).await,
698                Err(outcome) => refusal(outcome),
699            };
700            guard.finished = true;
701            drop(execution_hold);
702            outcome
703        });
704        drop(gate);
705        Ok(BackgroundJob {
706            task: Some(task),
707            cancel,
708            cancel_on_drop: true,
709        })
710    }
711}
712impl AdmittedJob {
713    pub fn class(&self) -> WorkClass {
714        self.lease.class
715    }
716    pub fn is_cancelled(&self) -> bool {
717        self.lease.cancel.is_cancelled()
718    }
719    pub fn request_cancel(&self) {
720        self.lease.cancel.request();
721    }
722    pub async fn cancelled(&self) {
723        let mut changed = self.lease.cancel.domain.changed.subscribe();
724        while !self.is_cancelled() {
725            if changed.changed().await.is_err() {
726                return;
727            }
728        }
729    }
730    pub async fn run_body<F: Future>(&self, body: F) -> BodyOutcome<F::Output> {
731        if self.is_cancelled() {
732            return BodyOutcome::Cancelled;
733        }
734        tokio::select! {
735            biased;
736            result = body => BodyOutcome::Finished(result),
737            _ = self.cancelled() => BodyOutcome::Cancelled,
738        }
739    }
740    pub async fn finalize<F: Future<Output = ()>>(&self, append: F) -> FinalizationOutcome {
741        let cancel = &self.lease.cancel;
742        if cancel
743            .finalization
744            .compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst)
745            .is_err()
746        {
747            return FinalizationOutcome::AlreadyFinalized;
748        }
749        let mut changed = cancel.domain.changed.subscribe();
750        let outcome = if cancel.deadline_overflow.load(Ordering::SeqCst) {
751            FinalizationOutcome::DeadlineOverflow
752        } else {
753            tokio::pin!(append);
754            let mut polled = false;
755            loop {
756                let deadline = match cancel.effective_deadline() {
757                    Ok(deadline) => deadline,
758                    Err(_) => break FinalizationOutcome::DeadlineOverflow,
759                };
760                let remaining = deadline.saturating_duration_since(Instant::now());
761                if remaining.is_zero() {
762                    break if polled {
763                        FinalizationOutcome::AttemptTimedOut
764                    } else {
765                        FinalizationOutcome::NoAttemptBeforeDeadline
766                    };
767                }
768                // Claiming the attempt precedes its first poll; deadline wakeups
769                // only shorten this same future, never construct a retry.
770                polled = true;
771                tokio::select! {
772                    biased;
773                    _ = &mut append => break FinalizationOutcome::AttemptReturned,
774                    _ = changed.changed() => {},
775                    _ = tokio::time::sleep(remaining) => {},
776                }
777            }
778        };
779        let incomplete = !matches!(outcome, FinalizationOutcome::AttemptReturned);
780        cancel
781            .finalization
782            .store(if incomplete { 3 } else { 2 }, Ordering::SeqCst);
783        if incomplete {
784            cancel.domain.incomplete.fetch_add(1, Ordering::SeqCst);
785        }
786        outcome
787    }
788    pub fn no_telemetry(&self) -> FinalizationOutcome {
789        if self
790            .lease
791            .cancel
792            .finalization
793            .compare_exchange(0, 2, Ordering::SeqCst, Ordering::SeqCst)
794            .is_ok()
795        {
796            FinalizationOutcome::NoTelemetryToken
797        } else {
798            FinalizationOutcome::AlreadyFinalized
799        }
800    }
801    pub fn spawn_blocking<F, T>(&self, body: F) -> Result<BackgroundJob<T>, ResourceError>
802    where
803        F: FnOnce() -> T + Send + 'static,
804        T: Send + 'static,
805    {
806        let domain = self.lease.cancel.domain.clone();
807        let gate = lock(&domain.gate);
808        if gate.closed {
809            return Err(domain.closed_error());
810        }
811        let cancel = self.lease.cancel.clone();
812        // Tokio can enqueue before panicking on worker creation. Only this
813        // recoverable escrow owns the user data and child lease. The inert
814        // queued wrapper owns neither after failed handoff reclamation.
815        let escrow = Arc::new(Mutex::new(BlockingEscrow {
816            published: false,
817            work: Some(BlockingWork {
818                body,
819                child: self.clone(),
820            }),
821        }));
822        let queued_escrow = escrow.clone();
823        let queued_domain = domain.clone();
824        // Catch *inside* the gate: a dequeued worker must not start user code
825        // between submission unwind and failed-publication/reclamation.
826        let submitted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
827            let task = self.lease.executor.spawn_blocking(move || {
828                #[cfg(test)]
829                if let Some(dequeued) = lock(&queued_domain.blocking_dequeued).take() {
830                    let _ = dequeued.send(());
831                }
832                let gate = lock(&queued_domain.gate);
833                let work = {
834                    let mut escrow = lock(&queued_escrow);
835                    if escrow.published {
836                        escrow.work.take()
837                    } else {
838                        None
839                    }
840                };
841                let Some(BlockingWork { body, child }) = work else {
842                    // Failed submit already reclaimed all user/lease ownership.
843                    return JobOutcome::Cancelled;
844                };
845                let decision = start_decision_locked(&child.lease.cancel, &gate);
846                drop(gate);
847                // Run AND drop user data outside the gate: destructors may
848                // reenter domain control. Started bodies keep the actual hold.
849                let outcome = match decision {
850                    Ok(()) => JobOutcome::Completed(body()),
851                    Err(outcome) => {
852                        drop(body);
853                        refusal(outcome)
854                    }
855                };
856                drop(child);
857                outcome
858            });
859            #[cfg(test)]
860            {
861                let fault = lock(&domain.after_blocking_enqueue).take();
862                if let Some(fault) = fault {
863                    fault();
864                }
865            }
866            task
867        }));
868        match submitted {
869            Ok(task) => {
870                lock(&escrow).published = true;
871                drop(gate);
872                Ok(BackgroundJob {
873                    task: Some(task),
874                    cancel,
875                    cancel_on_drop: true,
876                })
877            }
878            Err(payload) => {
879                let unstarted = lock(&escrow).work.take();
880                // published remains false while holding the start gate. The
881                // worker cannot take the reclaimed work after we unlock.
882                drop(gate);
883                drop(unstarted);
884                // Panic payloads may also have user-defined destructors.
885                drop(payload);
886                Err(ResourceError::ThreadStart(
887                    "blocking task submission panicked before handoff".into(),
888                ))
889            }
890        }
891    }
892}
893
894struct BlockingWork<F> {
895    // Field drop order keeps the child hold until captured user data is gone.
896    body: F,
897    child: AdmittedJob,
898}
899struct BlockingEscrow<F> {
900    // Accessed only under the domain start gate, then the escrow mutex.
901    published: bool,
902    work: Option<BlockingWork<F>>,
903}
904
905pub struct BackgroundJob<T> {
906    task: Option<JoinHandle<JobOutcome<T>>>,
907    cancel: Arc<JobCancellation>,
908    cancel_on_drop: bool,
909}
910impl<T> BackgroundJob<T> {
911    pub fn request_cancel(&self) {
912        self.cancel.request();
913    }
914    pub async fn join(mut self) -> Result<JobOutcome<T>, BackgroundJoinError> {
915        let result = self.task.as_mut().expect("join consumes one task").await;
916        self.cancel_on_drop = false;
917        result.map_err(|error| {
918            if error.is_cancelled() {
919                BackgroundJoinError::Cancelled
920            } else if error.is_panic() {
921                BackgroundJoinError::Panicked
922            } else {
923                BackgroundJoinError::Other(error.to_string())
924            }
925        })
926    }
927    pub fn detach(mut self) {
928        self.cancel_on_drop = false;
929    }
930}
931impl<T> Drop for BackgroundJob<T> {
932    fn drop(&mut self) {
933        if self.cancel_on_drop {
934            self.cancel.request();
935        }
936        // Dropping Tokio's JoinHandle detaches. It must not abort the phase
937        // supervisor that still owns the bounded terminal-emission attempt.
938    }
939}