1use std::fmt;
6use std::future::Future;
7use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
8use std::sync::{Arc, Mutex, MutexGuard};
9use std::time::{Duration, Instant};
10use tokio::runtime::Handle;
11use tokio::sync::{watch, OwnedSemaphorePermit, Semaphore};
12use tokio::task::JoinHandle;
13
14mod priority;
15#[cfg(test)]
16mod tests;
17
18#[derive(Clone, Copy, Debug, Eq, PartialEq)]
19pub enum WorkClass {
20 Warm,
21 Maintenance,
22}
23
24#[derive(Clone, Copy, Debug, Eq, PartialEq)]
25pub struct WorkClassLimits {
26 warm: usize,
27 maintenance: usize,
28}
29impl WorkClassLimits {
30 pub fn validate(warm: usize, maintenance: usize) -> Result<Self, ResourceError> {
31 for (class, value) in [
32 (WorkClass::Warm, warm),
33 (WorkClass::Maintenance, maintenance),
34 ] {
35 if value == 0 || value > Semaphore::MAX_PERMITS {
36 return Err(ResourceError::InvalidLimit {
37 class,
38 value,
39 max: Semaphore::MAX_PERMITS,
40 });
41 }
42 }
43 Ok(Self { warm, maintenance })
44 }
45 pub fn warm(self) -> usize {
46 self.warm
47 }
48 pub fn maintenance(self) -> usize {
49 self.maintenance
50 }
51}
52
53#[derive(Clone, Copy, Debug)]
54pub struct CleanupPolicy {
55 grace: Duration,
56}
57impl CleanupPolicy {
58 pub fn new(grace: Duration) -> Self {
59 Self { grace }
60 }
61 pub fn grace(self) -> Duration {
62 self.grace
63 }
64 pub fn deadline_from(self, now: Instant) -> Result<Instant, ResourceError> {
65 now.checked_add(self.grace)
66 .ok_or(ResourceError::DeadlineOverflow)
67 }
68}
69
70#[derive(Clone, Debug)]
71pub enum PriorityError {
72 Unsupported {
73 target: &'static str,
74 },
75 Os {
76 operation: &'static str,
77 code: Option<i32>,
78 },
79 Unexpected {
80 operation: &'static str,
81 value: i32,
82 },
83}
84#[derive(Clone, Debug)]
85pub enum ResourceError {
86 InvalidLimit {
87 class: WorkClass,
88 value: usize,
89 max: usize,
90 },
91 DeadlineOverflow,
92 Closed,
93 WrongDomain,
94 Priority(PriorityError),
95 ThreadStart(String),
96 RuntimeStart(String),
97 CoordinatorGone,
98 JoinFailed(String),
99}
100#[derive(Debug)]
101pub enum BackgroundJoinError {
102 Cancelled,
103 Panicked,
104 Other(String),
105}
106macro_rules! display_error {
107 ($($ty:ty),+ $(,)?) => {$ (
108 impl fmt::Display for $ty {
109 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self:?}") }
110 }
111 impl std::error::Error for $ty {}
112 )+ };
113}
114display_error!(PriorityError, ResourceError, BackgroundJoinError);
115
116#[derive(Debug)]
117pub enum BodyOutcome<T> {
118 Finished(T),
119 Cancelled,
120}
121#[derive(Debug)]
122pub enum JobOutcome<T> {
123 Completed(T),
124 Cancelled,
125 Refused(PriorityError),
126 FinalizationTimedOut,
127}
128#[derive(Clone, Copy, Debug, Eq, PartialEq)]
129pub enum FinalizationOutcome {
130 AttemptReturned,
131 AttemptTimedOut,
132 NoAttemptBeforeDeadline,
133 NoTelemetryToken,
134 AlreadyFinalized,
135 DeadlineOverflow,
136}
137#[derive(Clone, Debug)]
138pub struct ShutdownReport {
139 pub workers_exited: bool,
140 pub coordinator_joined: bool,
141 pub outstanding_jobs: usize,
142 pub incomplete_finalizations: usize,
143}
144
145fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
146 mutex
147 .lock()
148 .unwrap_or_else(std::sync::PoisonError::into_inner)
149}
150fn shorten(slot: &mut Option<Instant>, deadline: Instant) {
151 *slot = Some(slot.map_or(deadline, |old| old.min(deadline)));
152}
153struct StartGate {
154 closed: bool,
155 deadline: Option<Instant>,
156}
157struct JoinState {
158 coordinator: Option<std::thread::JoinHandle<()>>,
159 joining: bool,
160 joined: Option<Result<(), ResourceError>>,
161}
162struct Domain {
163 limits: WorkClassLimits,
164 cleanup: CleanupPolicy,
165 warm: Arc<Semaphore>,
166 maintenance: Arc<Semaphore>,
167 gate: Mutex<StartGate>,
168 changed: watch::Sender<()>,
169 supervisors: AtomicUsize,
170 jobs: AtomicUsize,
171 incomplete: AtomicUsize,
172 worker_exit: Mutex<Option<Result<(), ResourceError>>>,
173 join: Mutex<JoinState>,
174 posture_failure: Mutex<Option<PriorityError>>,
175 #[cfg(test)]
177 after_blocking_enqueue: Mutex<Option<Box<dyn FnOnce() + Send>>>,
178 #[cfg(test)]
179 blocking_dequeued: Mutex<Option<std::sync::mpsc::Sender<()>>>,
180}
181impl Domain {
182 fn new(limits: WorkClassLimits, cleanup: CleanupPolicy) -> Arc<Self> {
183 let (changed, _) = watch::channel(());
184 Arc::new(Self {
185 limits,
186 cleanup,
187 warm: Arc::new(Semaphore::new(limits.warm)),
188 maintenance: Arc::new(Semaphore::new(limits.maintenance)),
189 gate: Mutex::new(StartGate {
190 closed: false,
191 deadline: None,
192 }),
193 changed,
194 supervisors: AtomicUsize::new(0),
195 jobs: AtomicUsize::new(0),
196 incomplete: AtomicUsize::new(0),
197 worker_exit: Mutex::new(None),
198 join: Mutex::new(JoinState {
199 coordinator: None,
200 joining: false,
201 joined: None,
202 }),
203 posture_failure: Mutex::new(None),
204 #[cfg(test)]
205 after_blocking_enqueue: Mutex::new(None),
206 #[cfg(test)]
207 blocking_dequeued: Mutex::new(None),
208 })
209 }
210 fn notify(&self) {
211 let _ = self.changed.send(());
212 }
213 fn close(&self, deadline: Instant) {
214 {
215 let mut gate = lock(&self.gate);
216 gate.closed = true;
217 shorten(&mut gate.deadline, deadline);
218 self.warm.close();
219 self.maintenance.close();
220 }
221 self.notify();
222 }
223 fn close_with_policy(&self) -> Result<(), ResourceError> {
224 let now = Instant::now();
225 match self.cleanup.deadline_from(now) {
226 Ok(deadline) => {
227 self.close(deadline);
228 Ok(())
229 }
230 Err(error) => {
231 self.close(now);
232 Err(error)
233 }
234 }
235 }
236 fn closed_error(&self) -> ResourceError {
237 lock(&self.posture_failure)
238 .clone()
239 .map_or(ResourceError::Closed, ResourceError::Priority)
240 }
241 fn posture_failed(&self, error: PriorityError) {
242 tracing::error!(error = %error, "isolated background worker priority failed; closing domain");
243 let mut failure = lock(&self.posture_failure);
244 if failure.is_none() {
245 *failure = Some(error);
246 }
247 drop(failure);
248 let _ = self.close_with_policy();
249 }
250 fn report(&self) -> ShutdownReport {
251 ShutdownReport {
252 workers_exited: lock(&self.worker_exit).is_some(),
253 coordinator_joined: lock(&self.join).joined.as_ref().is_some_and(Result::is_ok),
254 outstanding_jobs: self.jobs.load(Ordering::SeqCst),
255 incomplete_finalizations: self.incomplete.load(Ordering::SeqCst),
256 }
257 }
258 async fn drive_until_closed(&self) {
259 let mut changed = self.changed.subscribe();
260 loop {
261 let (closed, deadline) = {
262 let gate = lock(&self.gate);
263 (gate.closed, gate.deadline)
264 };
265 if closed {
266 if self.supervisors.load(Ordering::SeqCst) == 0 {
267 return;
268 }
269 let remaining = deadline
270 .unwrap_or_else(Instant::now)
271 .saturating_duration_since(Instant::now());
272 if remaining.is_zero() {
273 return;
274 }
275 tokio::select! {
276 _ = changed.changed() => {},
277 _ = tokio::time::sleep(remaining) => {},
278 }
279 } else if changed.changed().await.is_err() {
280 return;
281 }
282 }
283 }
284}
285
286struct ExternalOwner {
287 domain: Arc<Domain>,
288 executor: Handle,
289}
290impl Drop for ExternalOwner {
291 fn drop(&mut self) {
292 let _ = self.domain.close_with_policy();
293 }
294}
295#[derive(Clone)]
296pub struct BackgroundRoot {
297 owner: Arc<ExternalOwner>,
298}
299pub type BackgroundDomainOwner = BackgroundRoot;
300#[derive(Clone)]
301pub struct BackgroundDomainHandle {
302 domain: Arc<Domain>,
303 executor: Handle,
304}
305#[derive(Clone)]
306pub struct BackgroundDomainControl {
307 domain: Arc<Domain>,
308}
309
310impl BackgroundRoot {
311 pub fn start(limits: WorkClassLimits, cleanup: CleanupPolicy) -> Result<Self, ResourceError> {
312 Self::start_inner(limits, cleanup, None)
313 }
314 fn start_inner(
315 limits: WorkClassLimits,
316 cleanup: CleanupPolicy,
317 blocking_threads: Option<usize>,
318 ) -> Result<Self, ResourceError> {
319 cleanup.deadline_from(Instant::now())?;
321 let domain = Domain::new(limits, cleanup);
322 let worker_domain = domain.clone();
323 let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1);
324 let coordinator = std::thread::Builder::new()
325 .name("khive-background".into())
326 .spawn(move || {
327 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
328 if let Err(error) = priority::lower_current_thread() {
329 let error = ResourceError::Priority(error);
330 let _ = ready_tx.send(Err(error.clone()));
331 return Err(error);
332 }
333 let hook_domain = worker_domain.clone();
334 let mut builder = tokio::runtime::Builder::new_current_thread();
335 if let Some(count) = blocking_threads {
336 builder.max_blocking_threads(count);
337 }
338 let runtime = match builder
339 .enable_all()
340 .on_thread_start(move || {
341 if let Err(error) = priority::lower_current_thread() {
342 hook_domain.posture_failed(error);
343 }
344 })
345 .build()
346 {
347 Ok(runtime) => runtime,
348 Err(error) => {
349 let error = ResourceError::RuntimeStart(error.to_string());
350 let _ = ready_tx.send(Err(error.clone()));
351 return Err(error);
352 }
353 };
354 if ready_tx.send(Ok(runtime.handle().clone())).is_err() {
355 worker_domain.close(Instant::now());
356 }
357 runtime.block_on(worker_domain.drive_until_closed());
358 drop(runtime);
361 Ok(())
362 }))
363 .unwrap_or_else(|_| Err(ResourceError::JoinFailed("coordinator panicked".into())));
364 *lock(&worker_domain.worker_exit) = Some(result);
365 worker_domain.notify();
366 })
367 .map_err(|error| ResourceError::ThreadStart(error.to_string()))?;
368 lock(&domain.join).coordinator = Some(coordinator);
369 let executor = ready_rx
370 .recv()
371 .map_err(|_| ResourceError::CoordinatorGone)??;
372 Ok(Self {
373 owner: Arc::new(ExternalOwner { domain, executor }),
374 })
375 }
376 pub fn handle(&self) -> BackgroundDomainHandle {
377 BackgroundDomainHandle {
378 domain: self.owner.domain.clone(),
379 executor: self.owner.executor.clone(),
380 }
381 }
382 pub fn control(&self) -> BackgroundDomainControl {
383 self.handle().control()
384 }
385 pub fn limits(&self) -> WorkClassLimits {
386 self.owner.domain.limits
387 }
388 pub fn cleanup_policy(&self) -> CleanupPolicy {
389 self.owner.domain.cleanup
390 }
391 pub fn same_domain(&self, other: &Self) -> bool {
392 Arc::ptr_eq(&self.owner.domain, &other.owner.domain)
393 }
394}
395
396impl BackgroundDomainControl {
397 pub async fn closed(&self) {
401 let mut changed = self.domain.changed.subscribe();
402 while !self.is_closed() {
403 let _ = changed.changed().await;
406 }
407 }
408 pub fn close(&self, deadline: Instant) {
409 self.domain.close(deadline);
410 }
411 pub fn close_with_policy(&self) -> Result<(), ResourceError> {
412 self.domain.close_with_policy()
413 }
414 pub fn is_closed(&self) -> bool {
415 lock(&self.domain.gate).closed
416 }
417 fn ensure_join_started(&self) -> Result<(), ResourceError> {
418 let mut join = lock(&self.domain.join);
419 if join.joining || join.joined.is_some() {
420 return Ok(());
421 }
422 let Some(coordinator) = join.coordinator.take() else {
423 return Err(ResourceError::CoordinatorGone);
424 };
425 let envelope = Arc::new(Mutex::new(Some(coordinator)));
428 let envelope_worker = envelope.clone();
429 let domain = self.domain.clone();
430 join.joining = true;
431 let spawned = std::thread::Builder::new()
432 .name("khive-background-join".into())
433 .spawn(move || {
434 let coordinator = lock(&envelope_worker)
435 .take()
436 .expect("single reaper owns coordinator");
437 let result = coordinator
438 .join()
439 .map_err(|_| ResourceError::JoinFailed("coordinator panicked".into()))
440 .and_then(|()| {
441 lock(&domain.worker_exit)
442 .clone()
443 .unwrap_or(Err(ResourceError::CoordinatorGone))
444 });
445 let mut join = lock(&domain.join);
446 join.joined = Some(result);
447 join.joining = false;
448 drop(join);
449 domain.notify();
450 });
451 match spawned {
452 Ok(reaper) => {
453 drop(reaper);
454 Ok(())
455 }
456 Err(error) => {
457 join.coordinator = lock(&envelope).take();
458 join.joining = false;
459 Err(ResourceError::ThreadStart(error.to_string()))
460 }
461 }
462 }
463 pub async fn observe_exit_until(
467 &self,
468 deadline: Instant,
469 ) -> Result<ShutdownReport, ResourceError> {
470 self.ensure_join_started()?;
471 let mut changed = self.domain.changed.subscribe();
472 loop {
473 let joined = { lock(&self.domain.join).joined.clone() };
474 if let Some(result) = joined {
475 result?;
476 return Ok(self.domain.report());
477 }
478 let remaining = deadline.saturating_duration_since(Instant::now());
479 if remaining.is_zero() {
480 return Ok(self.domain.report());
481 }
482 tokio::select! {
483 _ = changed.changed() => {},
484 _ = tokio::time::sleep(remaining) => {},
485 }
486 }
487 }
488 pub async fn shutdown(&self, deadline: Instant) -> Result<ShutdownReport, ResourceError> {
489 self.close(deadline);
490 self.ensure_join_started()?;
491 let mut changed = self.domain.changed.subscribe();
492 loop {
493 let joined = { lock(&self.domain.join).joined.clone() };
494 if let Some(result) = joined {
495 result?;
496 return Ok(self.domain.report());
497 }
498 let effective = lock(&self.domain.gate).deadline.unwrap_or(deadline);
501 let remaining = effective.saturating_duration_since(Instant::now());
502 if remaining.is_zero() {
503 return Ok(self.domain.report());
504 }
505 tokio::select! {
506 _ = changed.changed() => {},
507 _ = tokio::time::sleep(remaining) => {},
508 }
509 }
510 }
511}
512
513#[derive(Clone)]
514pub struct TrackedJob {
515 _guard: Arc<crate::daemon::BackgroundTaskGuard>,
516}
517impl TrackedJob {
518 pub fn register(name: &'static str) -> Self {
519 Self {
520 _guard: Arc::new(crate::daemon::register_background_task(name)),
521 }
522 }
523}
524struct JobCancellation {
525 domain: Arc<Domain>,
526 requested: AtomicBool,
527 deadline: Mutex<Option<Instant>>,
528 deadline_overflow: AtomicBool,
529 finalization: AtomicU8,
531}
532impl JobCancellation {
533 fn request(&self) {
534 let gate = lock(&self.domain.gate);
535 if !self.requested.swap(true, Ordering::SeqCst) {
536 let now = Instant::now();
537 let deadline = self.domain.cleanup.deadline_from(now).unwrap_or_else(|_| {
538 self.deadline_overflow.store(true, Ordering::SeqCst);
539 now
540 });
541 shorten(&mut lock(&self.deadline), deadline);
542 }
543 drop(gate);
544 self.domain.notify();
545 }
546 fn is_cancelled(&self) -> bool {
547 lock(&self.domain.gate).closed || self.requested.load(Ordering::SeqCst)
548 }
549 fn effective_deadline(&self) -> Result<Instant, ResourceError> {
550 let gate = lock(&self.domain.gate);
551 let mut own = lock(&self.deadline);
552 if own.is_none() {
553 *own = Some(self.domain.cleanup.deadline_from(Instant::now())?);
554 }
555 Ok(gate
556 .deadline
557 .map_or(own.expect("deadline initialized"), |d| {
558 d.min(own.expect("deadline initialized"))
559 }))
560 }
561}
562struct Lease {
563 _permit: OwnedSemaphorePermit,
564 _tracking: TrackedJob,
565 cancel: Arc<JobCancellation>,
566 executor: Handle,
567 class: WorkClass,
568}
569impl Drop for Lease {
570 fn drop(&mut self) {
571 self.cancel.domain.jobs.fetch_sub(1, Ordering::SeqCst);
572 self.cancel.domain.notify();
573 }
574}
575#[derive(Clone)]
576pub struct AdmittedJob {
577 lease: Arc<Lease>,
578}
579
580struct SupervisorGuard {
581 cancel: Arc<JobCancellation>,
582 finished: bool,
583}
584impl Drop for SupervisorGuard {
585 fn drop(&mut self) {
586 let phase = self.cancel.finalization.load(Ordering::SeqCst);
587 if (!self.finished && phase < 2) || phase == 1 {
588 self.cancel.domain.incomplete.fetch_add(1, Ordering::SeqCst);
589 }
590 self.cancel
591 .domain
592 .supervisors
593 .fetch_sub(1, Ordering::SeqCst);
594 self.cancel.domain.notify();
595 }
596}
597fn start_decision(cancel: &JobCancellation) -> Result<(), JobOutcome<()>> {
598 let gate = lock(&cancel.domain.gate);
599 start_decision_locked(cancel, &gate)
600}
601fn start_decision_locked(cancel: &JobCancellation, gate: &StartGate) -> Result<(), JobOutcome<()>> {
602 if let Some(error) = lock(&cancel.domain.posture_failure).clone() {
603 return Err(JobOutcome::Refused(error));
604 }
605 if gate.closed || cancel.requested.load(Ordering::SeqCst) {
606 return Err(JobOutcome::Cancelled);
607 }
608 if let Err(error) = priority::check_current_thread() {
609 return Err(JobOutcome::Refused(error));
610 }
611 Ok(())
614}
615fn refusal<T>(outcome: JobOutcome<()>) -> JobOutcome<T> {
616 match outcome {
617 JobOutcome::Cancelled => JobOutcome::Cancelled,
618 JobOutcome::Refused(error) => JobOutcome::Refused(error),
619 _ => unreachable!("start decision only refuses cancellation or posture"),
620 }
621}
622impl BackgroundDomainHandle {
623 pub async fn closed(&self) {
625 self.control().closed().await;
626 }
627 pub fn limits(&self) -> WorkClassLimits {
628 self.domain.limits
629 }
630 pub fn control(&self) -> BackgroundDomainControl {
631 BackgroundDomainControl {
632 domain: self.domain.clone(),
633 }
634 }
635 pub async fn admit(
636 &self,
637 class: WorkClass,
638 tracking: &TrackedJob,
639 ) -> Result<AdmittedJob, ResourceError> {
640 let semaphore = match class {
641 WorkClass::Warm => self.domain.warm.clone(),
642 WorkClass::Maintenance => self.domain.maintenance.clone(),
643 };
644 let permit = semaphore
645 .acquire_owned()
646 .await
647 .map_err(|_| self.domain.closed_error())?;
648 let gate = lock(&self.domain.gate);
649 if gate.closed {
650 return Err(self.domain.closed_error());
651 }
652 self.domain.jobs.fetch_add(1, Ordering::SeqCst);
653 let cancel = Arc::new(JobCancellation {
654 domain: self.domain.clone(),
655 requested: AtomicBool::new(false),
656 deadline: Mutex::new(None),
657 deadline_overflow: AtomicBool::new(false),
658 finalization: AtomicU8::new(0),
659 });
660 Ok(AdmittedJob {
661 lease: Arc::new(Lease {
662 _permit: permit,
663 _tracking: tracking.clone(),
664 cancel,
665 executor: self.executor.clone(),
666 class,
667 }),
668 })
669 }
670 pub fn spawn_supervised<F, Fut, T>(
671 &self,
672 job: AdmittedJob,
673 make: F,
674 ) -> Result<BackgroundJob<T>, ResourceError>
675 where
676 F: FnOnce(AdmittedJob) -> Fut + Send + 'static,
677 Fut: Future<Output = JobOutcome<T>> + Send + 'static,
678 T: Send + 'static,
679 {
680 if !Arc::ptr_eq(&self.domain, &job.lease.cancel.domain) {
681 return Err(ResourceError::WrongDomain);
682 }
683 let gate = lock(&self.domain.gate);
684 if gate.closed {
685 return Err(self.domain.closed_error());
686 }
687 let cancel = job.lease.cancel.clone();
688 self.domain.supervisors.fetch_add(1, Ordering::SeqCst);
689 let guard = SupervisorGuard {
690 cancel: cancel.clone(),
691 finished: false,
692 };
693 let task = self.executor.spawn(async move {
694 let mut guard = guard;
695 let execution_hold = job.clone();
696 let outcome = match start_decision(&job.lease.cancel) {
697 Ok(()) => make(job).await,
698 Err(outcome) => refusal(outcome),
699 };
700 guard.finished = true;
701 drop(execution_hold);
702 outcome
703 });
704 drop(gate);
705 Ok(BackgroundJob {
706 task: Some(task),
707 cancel,
708 cancel_on_drop: true,
709 })
710 }
711}
712impl AdmittedJob {
713 pub fn class(&self) -> WorkClass {
714 self.lease.class
715 }
716 pub fn is_cancelled(&self) -> bool {
717 self.lease.cancel.is_cancelled()
718 }
719 pub fn request_cancel(&self) {
720 self.lease.cancel.request();
721 }
722 pub async fn cancelled(&self) {
723 let mut changed = self.lease.cancel.domain.changed.subscribe();
724 while !self.is_cancelled() {
725 if changed.changed().await.is_err() {
726 return;
727 }
728 }
729 }
730 pub async fn run_body<F: Future>(&self, body: F) -> BodyOutcome<F::Output> {
731 if self.is_cancelled() {
732 return BodyOutcome::Cancelled;
733 }
734 tokio::select! {
735 biased;
736 result = body => BodyOutcome::Finished(result),
737 _ = self.cancelled() => BodyOutcome::Cancelled,
738 }
739 }
740 pub async fn finalize<F: Future<Output = ()>>(&self, append: F) -> FinalizationOutcome {
741 let cancel = &self.lease.cancel;
742 if cancel
743 .finalization
744 .compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst)
745 .is_err()
746 {
747 return FinalizationOutcome::AlreadyFinalized;
748 }
749 let mut changed = cancel.domain.changed.subscribe();
750 let outcome = if cancel.deadline_overflow.load(Ordering::SeqCst) {
751 FinalizationOutcome::DeadlineOverflow
752 } else {
753 tokio::pin!(append);
754 let mut polled = false;
755 loop {
756 let deadline = match cancel.effective_deadline() {
757 Ok(deadline) => deadline,
758 Err(_) => break FinalizationOutcome::DeadlineOverflow,
759 };
760 let remaining = deadline.saturating_duration_since(Instant::now());
761 if remaining.is_zero() {
762 break if polled {
763 FinalizationOutcome::AttemptTimedOut
764 } else {
765 FinalizationOutcome::NoAttemptBeforeDeadline
766 };
767 }
768 polled = true;
771 tokio::select! {
772 biased;
773 _ = &mut append => break FinalizationOutcome::AttemptReturned,
774 _ = changed.changed() => {},
775 _ = tokio::time::sleep(remaining) => {},
776 }
777 }
778 };
779 let incomplete = !matches!(outcome, FinalizationOutcome::AttemptReturned);
780 cancel
781 .finalization
782 .store(if incomplete { 3 } else { 2 }, Ordering::SeqCst);
783 if incomplete {
784 cancel.domain.incomplete.fetch_add(1, Ordering::SeqCst);
785 }
786 outcome
787 }
788 pub fn no_telemetry(&self) -> FinalizationOutcome {
789 if self
790 .lease
791 .cancel
792 .finalization
793 .compare_exchange(0, 2, Ordering::SeqCst, Ordering::SeqCst)
794 .is_ok()
795 {
796 FinalizationOutcome::NoTelemetryToken
797 } else {
798 FinalizationOutcome::AlreadyFinalized
799 }
800 }
801 pub fn spawn_blocking<F, T>(&self, body: F) -> Result<BackgroundJob<T>, ResourceError>
802 where
803 F: FnOnce() -> T + Send + 'static,
804 T: Send + 'static,
805 {
806 let domain = self.lease.cancel.domain.clone();
807 let gate = lock(&domain.gate);
808 if gate.closed {
809 return Err(domain.closed_error());
810 }
811 let cancel = self.lease.cancel.clone();
812 let escrow = Arc::new(Mutex::new(BlockingEscrow {
816 published: false,
817 work: Some(BlockingWork {
818 body,
819 child: self.clone(),
820 }),
821 }));
822 let queued_escrow = escrow.clone();
823 let queued_domain = domain.clone();
824 let submitted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
827 let task = self.lease.executor.spawn_blocking(move || {
828 #[cfg(test)]
829 if let Some(dequeued) = lock(&queued_domain.blocking_dequeued).take() {
830 let _ = dequeued.send(());
831 }
832 let gate = lock(&queued_domain.gate);
833 let work = {
834 let mut escrow = lock(&queued_escrow);
835 if escrow.published {
836 escrow.work.take()
837 } else {
838 None
839 }
840 };
841 let Some(BlockingWork { body, child }) = work else {
842 return JobOutcome::Cancelled;
844 };
845 let decision = start_decision_locked(&child.lease.cancel, &gate);
846 drop(gate);
847 let outcome = match decision {
850 Ok(()) => JobOutcome::Completed(body()),
851 Err(outcome) => {
852 drop(body);
853 refusal(outcome)
854 }
855 };
856 drop(child);
857 outcome
858 });
859 #[cfg(test)]
860 {
861 let fault = lock(&domain.after_blocking_enqueue).take();
862 if let Some(fault) = fault {
863 fault();
864 }
865 }
866 task
867 }));
868 match submitted {
869 Ok(task) => {
870 lock(&escrow).published = true;
871 drop(gate);
872 Ok(BackgroundJob {
873 task: Some(task),
874 cancel,
875 cancel_on_drop: true,
876 })
877 }
878 Err(payload) => {
879 let unstarted = lock(&escrow).work.take();
880 drop(gate);
883 drop(unstarted);
884 drop(payload);
886 Err(ResourceError::ThreadStart(
887 "blocking task submission panicked before handoff".into(),
888 ))
889 }
890 }
891 }
892}
893
894struct BlockingWork<F> {
895 body: F,
897 child: AdmittedJob,
898}
899struct BlockingEscrow<F> {
900 published: bool,
902 work: Option<BlockingWork<F>>,
903}
904
905pub struct BackgroundJob<T> {
906 task: Option<JoinHandle<JobOutcome<T>>>,
907 cancel: Arc<JobCancellation>,
908 cancel_on_drop: bool,
909}
910impl<T> BackgroundJob<T> {
911 pub fn request_cancel(&self) {
912 self.cancel.request();
913 }
914 pub async fn join(mut self) -> Result<JobOutcome<T>, BackgroundJoinError> {
915 let result = self.task.as_mut().expect("join consumes one task").await;
916 self.cancel_on_drop = false;
917 result.map_err(|error| {
918 if error.is_cancelled() {
919 BackgroundJoinError::Cancelled
920 } else if error.is_panic() {
921 BackgroundJoinError::Panicked
922 } else {
923 BackgroundJoinError::Other(error.to_string())
924 }
925 })
926 }
927 pub fn detach(mut self) {
928 self.cancel_on_drop = false;
929 }
930}
931impl<T> Drop for BackgroundJob<T> {
932 fn drop(&mut self) {
933 if self.cancel_on_drop {
934 self.cancel.request();
935 }
936 }
939}