Skip to main content

rapidgzip_core/
pool.rs

1//! Process-wide decode execution-slot allocation.
2
3use std::collections::VecDeque;
4use std::error::Error;
5use std::fmt::{self, Display, Formatter};
6use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
7use std::sync::{Arc, Condvar, Mutex, Weak};
8
9/// Invalid [`DecoderPool`] configuration.
10#[derive(Clone, Copy, Debug, Eq, PartialEq)]
11pub struct DecoderPoolConfigError(&'static str);
12
13impl Display for DecoderPoolConfigError {
14    fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
15        formatter.write_str(self.0)
16    }
17}
18
19impl Error for DecoderPoolConfigError {}
20
21/// Invalid runtime worker limit for a [`DecoderPool`].
22#[derive(Clone, Copy, Debug, Eq, PartialEq)]
23pub struct DecoderPoolLimitError {
24    requested: usize,
25    configured: usize,
26}
27
28impl DecoderPoolLimitError {
29    /// Rejected worker count.
30    pub const fn requested(self) -> usize {
31        self.requested
32    }
33
34    /// Immutable maximum configured for the pool.
35    pub const fn configured(self) -> usize {
36        self.configured
37    }
38}
39
40impl Display for DecoderPoolLimitError {
41    fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
42        write!(
43            formatter,
44            "pool worker limit {} is outside 1..={}",
45            self.requested, self.configured
46        )
47    }
48}
49
50impl Error for DecoderPoolLimitError {}
51
52/// Approximate process-wide snapshot of a [`DecoderPool`].
53///
54/// Fields are sampled independently. The member list is held briefly while
55/// aggregate queue depth is summed, but the snapshot is not a transactionally
56/// consistent view of a single instant. Counts describe current state, not
57/// lifetime maxima. In particular, `spawned_workers` can exceed `worker_limit`
58/// while threads retire after a decrease, and `busy_workers` can temporarily
59/// exceed a newly lowered limit while already-admitted tasks finish.
60#[derive(Clone, Copy, Debug, Eq, PartialEq)]
61#[non_exhaustive]
62pub struct DecoderPoolStats {
63    /// Immutable maximum number of simultaneous decode execution slots.
64    pub configured_workers: usize,
65    /// Current application-controlled pool ceiling.
66    pub worker_limit: usize,
67    /// Decode tasks currently holding a pool execution slot.
68    pub busy_workers: usize,
69    /// Worker allowances currently distributed across attached decoders.
70    ///
71    /// An allowance permits a decoder to keep a worker representative; it is
72    /// not an occupied execution slot. Compare with [`Self::busy_workers`].
73    pub active_workers: usize,
74    /// Live decoder-worker operating-system threads attached to the pool.
75    pub spawned_workers: usize,
76    /// Live coordinator and scanner threads attached to the pool.
77    pub auxiliary_threads: usize,
78    /// Approximate runnable tasks across attached decoders.
79    pub queued_tasks: usize,
80    /// Live decoder operations attached to the pool.
81    pub attached_decoders: usize,
82    /// Attached decoders with queued, executing, or pool-waiting work.
83    pub runnable_decoders: usize,
84    /// Attached decoders with at least one task waiting for a pool slot.
85    pub waiting_decoders: usize,
86}
87
88/// Cloneable process-wide budget shared by multiple decoders.
89///
90/// The pool controls decode *execution slots*. It deliberately does not expose
91/// or promise a permanent one-to-one mapping between slots and operating-system
92/// threads. Each decoder retains its format-specific worker loop and scratch
93/// storage, while CPU-intensive tasks acquire a fair pool slot. A task releases
94/// its slot before any result or reader-channel operation that would block, so
95/// a slow consumer cannot occupy the shared decode budget.
96///
97/// The pool is opt-in and does not alter decoders built without
98/// [`crate::DecoderBuilder::decoder_pool`]. Its `workers` value is an immutable
99/// maximum; [`Self::set_worker_limit`] can change the live aggregate ceiling.
100/// Decoders retain their own per-operation maximum and adaptive policy, so a
101/// pool slot can remain unused when no attached input exposes useful work.
102///
103/// Under contention, stable max-min allowances prevent every decoder from
104/// spawning its complete configured headroom, while FIFO permit admission
105/// prevents one decoder from monopolizing execution. Cloning a pool shares the
106/// same state and does not create threads or reserve slots.
107///
108/// # Examples
109///
110/// ```no_run
111/// use rapidgzip_core::{Decoder, DecoderPool};
112/// use std::io;
113///
114/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
115/// let pool = DecoderPool::builder()
116///     .workers(24)
117///     .initial_worker_limit(8)
118///     .build()?;
119/// let decoder = Decoder::builder()
120///     .decoder_threads(24) // Per-operation headroom.
121///     .decoder_pool(pool.clone())
122///     .build()?;
123///
124/// let mut first = decoder.open("sample-R1.fastq.gz")?;
125/// let mut second = decoder.open("sample-R2.fastq.gz")?;
126/// first.request_workers(12)?;
127/// second.request_workers(12)?;
128/// pool.set_worker_limit(16)?; // Aggregate execution ceiling.
129///
130/// // Applications normally consume independent readers concurrently.
131/// io::copy(&mut first, &mut io::sink())?;
132/// io::copy(&mut second, &mut io::sink())?;
133/// # Ok(())
134/// # }
135/// ```
136#[derive(Clone)]
137pub struct DecoderPool {
138    pub(crate) state: Arc<PoolState>,
139}
140
141impl fmt::Debug for DecoderPool {
142    fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
143        formatter
144            .debug_struct("DecoderPool")
145            .field("stats", &self.stats())
146            .finish()
147    }
148}
149
150#[bon::bon]
151impl DecoderPool {
152    /// Creates a process-wide decoder pool.
153    ///
154    /// Use [`DecoderPool::builder`] for named configuration. `workers` is the
155    /// immutable maximum number of simultaneously occupied decode execution
156    /// slots. `initial_worker_limit` optionally starts the mutable aggregate
157    /// ceiling below that maximum so an external scheduler can grow it later.
158    /// Neither value eagerly creates operating-system threads.
159    ///
160    /// # Errors
161    ///
162    /// Returns [`DecoderPoolConfigError`] when `workers` is zero or
163    /// `initial_worker_limit` is zero or greater than `workers`.
164    #[builder]
165    pub fn new(
166        /// Immutable maximum number of simultaneous decode execution slots.
167        workers: usize,
168        /// Initial runtime ceiling; omitted means `workers`.
169        initial_worker_limit: Option<usize>,
170    ) -> Result<Self, DecoderPoolConfigError> {
171        if workers == 0 {
172            return Err(DecoderPoolConfigError("pool workers must be non-zero"));
173        }
174        let initial_worker_limit = initial_worker_limit.unwrap_or(workers);
175        if initial_worker_limit == 0 || initial_worker_limit > workers {
176            return Err(DecoderPoolConfigError(
177                "initial_worker_limit must be within 1..=workers",
178            ));
179        }
180        Ok(Self {
181            state: Arc::new(PoolState::new(workers, initial_worker_limit)),
182        })
183    }
184
185    /// Returns an approximate snapshot of aggregate pool activity.
186    ///
187    /// The small member registry is locked briefly to sum per-decoder queue
188    /// depths and allowances. No decoder task or pool permit is held while
189    /// waiting for this snapshot. The returned value is [`Copy`] and does not
190    /// retain or control the pool.
191    pub fn stats(&self) -> DecoderPoolStats {
192        self.state.stats()
193    }
194
195    /// Changes the maximum number of decode tasks that may execute at once.
196    ///
197    /// The operation is nonblocking. Existing tasks retain their slots until
198    /// they finish CPU-intensive work; no new task is admitted above the lower
199    /// limit. Raising the limit redistributes worker allowances and wakes
200    /// queued decoders immediately. The value is an aggregate ceiling, not a
201    /// request that the pool keep that many tasks busy.
202    ///
203    /// # Errors
204    ///
205    /// Returns [`DecoderPoolLimitError`] for zero or a value above the pool's
206    /// immutable configured maximum.
207    pub fn set_worker_limit(&self, workers: usize) -> Result<(), DecoderPoolLimitError> {
208        self.state.set_worker_limit(workers)
209    }
210}
211
212pub(crate) struct PoolState {
213    configured_workers: usize,
214    worker_limit: AtomicUsize,
215    limit_epoch: AtomicUsize,
216    busy_workers: AtomicUsize,
217    spawned_workers: AtomicUsize,
218    auxiliary_threads: AtomicUsize,
219    attached_decoders: AtomicUsize,
220    runnable_decoders: AtomicUsize,
221    waiting_decoders: AtomicUsize,
222    next_ticket: AtomicU64,
223    scheduler: Mutex<PoolScheduler>,
224    signal: Condvar,
225}
226
227impl PoolState {
228    fn new(configured_workers: usize, worker_limit: usize) -> Self {
229        Self {
230            configured_workers,
231            worker_limit: AtomicUsize::new(worker_limit),
232            limit_epoch: AtomicUsize::new(0),
233            busy_workers: AtomicUsize::new(0),
234            spawned_workers: AtomicUsize::new(0),
235            auxiliary_threads: AtomicUsize::new(0),
236            attached_decoders: AtomicUsize::new(0),
237            runnable_decoders: AtomicUsize::new(0),
238            waiting_decoders: AtomicUsize::new(0),
239            next_ticket: AtomicU64::new(0),
240            scheduler: Mutex::new(PoolScheduler::default()),
241            signal: Condvar::new(),
242        }
243    }
244
245    pub(crate) const fn configured_workers(&self) -> usize {
246        self.configured_workers
247    }
248
249    fn set_worker_limit(&self, workers: usize) -> Result<(), DecoderPoolLimitError> {
250        if workers == 0 || workers > self.configured_workers {
251            return Err(DecoderPoolLimitError {
252                requested: workers,
253                configured: self.configured_workers,
254            });
255        }
256        let previous = self.worker_limit.swap(workers, Ordering::Release);
257        if previous != workers {
258            self.limit_epoch.fetch_add(1, Ordering::Relaxed);
259            let mut scheduler = self.scheduler.lock().expect("decoder pool mutex poisoned");
260            self.rebalance(&mut scheduler, true);
261            self.signal.notify_all();
262        }
263        Ok(())
264    }
265
266    fn try_acquire_fast(&self) -> bool {
267        if self.waiting_decoders.load(Ordering::Acquire) != 0 {
268            return false;
269        }
270        let mut busy = self.busy_workers.load(Ordering::Relaxed);
271        loop {
272            let limit = self.worker_limit.load(Ordering::Acquire);
273            if busy >= limit {
274                return false;
275            }
276            match self.busy_workers.compare_exchange_weak(
277                busy,
278                busy + 1,
279                Ordering::AcqRel,
280                Ordering::Relaxed,
281            ) {
282                Ok(_) => return true,
283                Err(observed) => busy = observed,
284            }
285        }
286    }
287
288    fn acquire<'a>(&'a self, member: &'a PoolMemberState) -> PoolPermit<'a> {
289        if self.try_acquire_fast() {
290            let previous_busy = member.busy_workers.fetch_add(1, Ordering::Relaxed);
291            if previous_busy == 0 && !member.runnable.load(Ordering::Acquire) {
292                self.refresh_runnable(member);
293            }
294            return PoolPermit { pool: self, member };
295        }
296
297        let ticket = self.next_ticket.fetch_add(1, Ordering::Relaxed);
298        let first_for_decoder = member.waiting_workers.fetch_add(1, Ordering::AcqRel) == 0;
299        if first_for_decoder {
300            self.waiting_decoders.fetch_add(1, Ordering::Relaxed);
301            if !member.runnable.load(Ordering::Acquire) {
302                self.refresh_runnable(member);
303            }
304        }
305        let mut scheduler = self.scheduler.lock().expect("decoder pool mutex poisoned");
306        scheduler.waiters.push_back(ticket);
307        loop {
308            let is_front = scheduler
309                .waiters
310                .front()
311                .is_some_and(|front| *front == ticket);
312            let busy = self.busy_workers.load(Ordering::Relaxed);
313            let limit = self.worker_limit.load(Ordering::Acquire);
314            if is_front && busy < limit {
315                let popped = scheduler.waiters.pop_front();
316                debug_assert_eq!(popped, Some(ticket));
317                self.busy_workers.fetch_add(1, Ordering::AcqRel);
318                // Publish the busy task before removing its waiting state so
319                // the member remains continuously runnable to the allocator.
320                member.busy_workers.fetch_add(1, Ordering::Relaxed);
321                if member.waiting_workers.fetch_sub(1, Ordering::AcqRel) == 1 {
322                    self.waiting_decoders.fetch_sub(1, Ordering::Relaxed);
323                }
324                self.signal.notify_all();
325                return PoolPermit { pool: self, member };
326            }
327            scheduler = self
328                .signal
329                .wait(scheduler)
330                .expect("decoder pool mutex poisoned");
331        }
332    }
333
334    fn release(&self, member: &PoolMemberState) {
335        let previous_busy = member.busy_workers.fetch_sub(1, Ordering::Relaxed);
336        self.busy_workers.fetch_sub(1, Ordering::AcqRel);
337        if previous_busy == 1
338            && member.queued_tasks.load(Ordering::Relaxed) == 0
339            && member.waiting_workers.load(Ordering::Relaxed) == 0
340        {
341            self.refresh_runnable(member);
342        }
343        // The notify must be ordered against `acquire`'s check-then-wait by the
344        // scheduler mutex, or the wakeup can be lost: a waiter that has read
345        // `busy == limit` but not yet parked misses a notify sent now, then
346        // parks against a state nobody will announce again. Taking the lock
347        // (even briefly) forces this release to happen-before an in-flight
348        // check completes its park -- the waiter either sees the decremented
349        // count when it re-checks, or is already parked when the notify fires.
350        //
351        // Every other notify site already held the lock; this one, on the hot
352        // release path, skipped it as an optimisation. The cost of the lock is
353        // only ever paid when the slow path is active at all, and the saving
354        // bought a deadlock: two decoders sharing a shrunken pool park in
355        // `acquire`, the final release notifies nobody, and -- because
356        // `try_acquire_fast` refuses to jump a non-empty queue -- every future
357        // acquire joins the dead queue. Found as a salmon-side hang; the same
358        // pool and broker in piscem never hit it only because piscem's decode
359        // share keeps enough slots granted that the slow path is rarely
360        // entered.
361        drop(self.scheduler.lock().expect("decoder pool mutex poisoned"));
362        self.signal.notify_all();
363    }
364
365    pub(crate) fn register_decoder(self: &Arc<Self>) -> PoolMember {
366        self.attached_decoders.fetch_add(1, Ordering::Relaxed);
367        let state = Arc::new(PoolMemberState::default());
368        state.attached.store(true, Ordering::Relaxed);
369        // Until path selection publishes a measured demand, reserve broad
370        // headroom for this attached operation. This prevents the first of a
371        // concurrently opening group from consuming every spawn allowance
372        // merely because its format scan finished first.
373        state
374            .desired_workers
375            .store(self.configured_workers, Ordering::Relaxed);
376        let mut scheduler = self.scheduler.lock().expect("decoder pool mutex poisoned");
377        scheduler.members.push(Arc::downgrade(&state));
378        self.rebalance(&mut scheduler, true);
379        drop(scheduler);
380        PoolMember {
381            pool: Arc::clone(self),
382            state,
383        }
384    }
385
386    fn rebalance(&self, scheduler: &mut PoolScheduler, immediate_growth: bool) {
387        let members: Vec<_> = scheduler.members.iter().filter_map(Weak::upgrade).collect();
388        scheduler
389            .members
390            .retain(|member| member.strong_count() != 0);
391        for member in &members {
392            member.target_granted_workers.store(0, Ordering::Relaxed);
393        }
394
395        let candidates: Vec<_> = members
396            .iter()
397            .filter(|member| {
398                member.attached.load(Ordering::Acquire)
399                    && member.desired_workers.load(Ordering::Relaxed) != 0
400            })
401            .collect();
402        if candidates.is_empty() {
403            for member in members {
404                member.set_granted_workers(0);
405            }
406            return;
407        }
408
409        let candidate_count = candidates.len();
410        let mut remaining = self.worker_limit.load(Ordering::Acquire);
411        let mut granted = vec![0_usize; candidates.len()];
412        let start = scheduler.rotation % candidates.len();
413        while remaining != 0 {
414            let mut made_progress = false;
415            for offset in 0..candidates.len() {
416                let index = (start + offset) % candidates.len();
417                let desired = candidates[index].desired_workers.load(Ordering::Relaxed);
418                if granted[index] >= desired {
419                    continue;
420                }
421                granted[index] += 1;
422                remaining -= 1;
423                made_progress = true;
424                if remaining == 0 {
425                    break;
426                }
427            }
428            if !made_progress {
429                break;
430            }
431        }
432        for (member, grant) in candidates.into_iter().zip(granted) {
433            member
434                .target_granted_workers
435                .store(grant, Ordering::Relaxed);
436        }
437        for member in members {
438            let target = member.target_granted_workers.load(Ordering::Relaxed);
439            let current = member.granted_workers.load(Ordering::Relaxed);
440            if target <= current || immediate_growth {
441                member.set_granted_workers(target);
442                member.grant_growth_events.store(0, Ordering::Relaxed);
443            }
444        }
445        scheduler.rotation = (start + 1) % candidate_count;
446    }
447
448    /// Refreshes whether `member` has work that can consume an execution slot.
449    ///
450    /// Rebalancing occurs only when that boolean changes. Queue depths change
451    /// for every task, so avoiding a scheduler lock for nonzero-to-nonzero
452    /// updates keeps the private worker loops' hot publication path cheap.
453    fn refresh_runnable(&self, member: &PoolMemberState) {
454        let runnable = member.desired_workers.load(Ordering::Relaxed) != 0
455            && (member.queued_tasks.load(Ordering::Relaxed) != 0
456                || member.busy_workers.load(Ordering::Relaxed) != 0
457                || member.waiting_workers.load(Ordering::Relaxed) != 0);
458        let previous = member.runnable.swap(runnable, Ordering::AcqRel);
459        if previous == runnable {
460            return;
461        }
462        if runnable {
463            self.runnable_decoders.fetch_add(1, Ordering::Relaxed);
464        } else {
465            self.runnable_decoders.fetch_sub(1, Ordering::Relaxed);
466        }
467    }
468
469    fn stats(&self) -> DecoderPoolStats {
470        let (queued_tasks, active_workers) = self
471            .scheduler
472            .lock()
473            .expect("decoder pool mutex poisoned")
474            .members
475            .iter()
476            .filter_map(Weak::upgrade)
477            .fold((0_usize, 0_usize), |(queued, active), member| {
478                (
479                    queued.saturating_add(member.queued_tasks.load(Ordering::Relaxed)),
480                    active.saturating_add(member.granted_workers.load(Ordering::Relaxed)),
481                )
482            });
483        DecoderPoolStats {
484            configured_workers: self.configured_workers,
485            worker_limit: self.worker_limit.load(Ordering::Relaxed),
486            busy_workers: self.busy_workers.load(Ordering::Relaxed),
487            active_workers,
488            spawned_workers: self.spawned_workers.load(Ordering::Relaxed),
489            auxiliary_threads: self.auxiliary_threads.load(Ordering::Relaxed),
490            queued_tasks,
491            attached_decoders: self.attached_decoders.load(Ordering::Relaxed),
492            runnable_decoders: self.runnable_decoders.load(Ordering::Relaxed),
493            waiting_decoders: self.waiting_decoders.load(Ordering::Relaxed),
494        }
495    }
496}
497
498#[derive(Default)]
499struct PoolScheduler {
500    waiters: VecDeque<u64>,
501    members: Vec<Weak<PoolMemberState>>,
502    rotation: usize,
503}
504
505#[derive(Default)]
506pub(crate) struct PoolMemberState {
507    attached: AtomicBool,
508    queued_tasks: AtomicUsize,
509    busy_workers: AtomicUsize,
510    waiting_workers: AtomicUsize,
511    desired_workers: AtomicUsize,
512    target_granted_workers: AtomicUsize,
513    granted_workers: AtomicUsize,
514    grant_growth_events: AtomicUsize,
515    grant_epoch: AtomicUsize,
516    runnable: AtomicBool,
517}
518
519impl PoolMemberState {
520    fn set_granted_workers(&self, workers: usize) {
521        let previous = self.granted_workers.swap(workers, Ordering::Release);
522        if previous != workers {
523            self.grant_epoch.fetch_add(1, Ordering::Relaxed);
524        }
525    }
526}
527
528pub(crate) struct PoolMember {
529    pool: Arc<PoolState>,
530    state: Arc<PoolMemberState>,
531}
532
533impl PoolMember {
534    const GRANT_GROWTH_QUEUE_EVENTS: usize = 128;
535
536    pub(crate) fn acquire(&self) -> PoolPermit<'_> {
537        self.pool.acquire(&self.state)
538    }
539
540    #[cfg(test)]
541    pub(crate) fn busy_workers(&self) -> usize {
542        self.state.busy_workers.load(Ordering::Relaxed)
543    }
544
545    pub(crate) fn waiting_workers(&self) -> usize {
546        self.state.waiting_workers.load(Ordering::Relaxed)
547    }
548
549    pub(crate) fn queued_tasks(&self) -> usize {
550        self.state.queued_tasks.load(Ordering::Relaxed)
551    }
552
553    pub(crate) fn granted_workers(&self) -> usize {
554        self.state.granted_workers.load(Ordering::Acquire)
555    }
556
557    pub(crate) fn set_desired_workers(&self, count: usize) {
558        let previous = self.state.desired_workers.swap(count, Ordering::Relaxed);
559        if previous == count {
560            return;
561        }
562        self.pool.refresh_runnable(&self.state);
563        let mut scheduler = self
564            .pool
565            .scheduler
566            .lock()
567            .expect("decoder pool mutex poisoned");
568        self.pool.rebalance(&mut scheduler, count != 0);
569        self.pool.signal.notify_all();
570    }
571
572    pub(crate) fn set_queued_tasks(&self, count: usize) {
573        let previous = self.state.queued_tasks.swap(count, Ordering::Relaxed);
574        if (previous == 0) != (count == 0) {
575            self.pool.refresh_runnable(&self.state);
576        }
577        self.maybe_promote_grant();
578    }
579
580    /// Delays allocation growth caused only by a peer becoming terminal until
581    /// enough local queue activity remains to amortize additional OS threads.
582    /// Explicit demand changes and pool-limit increases bypass this filter.
583    fn maybe_promote_grant(&self) {
584        let target = self.state.target_granted_workers.load(Ordering::Acquire);
585        let current = self.state.granted_workers.load(Ordering::Relaxed);
586        if target <= current {
587            return;
588        }
589        let events = self
590            .state
591            .grant_growth_events
592            .fetch_add(1, Ordering::Relaxed)
593            + 1;
594        if events < Self::GRANT_GROWTH_QUEUE_EVENTS {
595            return;
596        }
597
598        let target = self.state.target_granted_workers.load(Ordering::Acquire);
599        let mut current = self.state.granted_workers.load(Ordering::Relaxed);
600        while current < target {
601            match self.state.granted_workers.compare_exchange_weak(
602                current,
603                target,
604                Ordering::AcqRel,
605                Ordering::Relaxed,
606            ) {
607                Ok(_) => {
608                    self.state.grant_epoch.fetch_add(1, Ordering::Relaxed);
609                    break;
610                }
611                Err(observed) => current = observed,
612            }
613        }
614        self.state.grant_growth_events.store(0, Ordering::Relaxed);
615    }
616
617    pub(crate) fn register_worker(&self) {
618        self.pool.spawned_workers.fetch_add(1, Ordering::Relaxed);
619    }
620
621    pub(crate) fn unregister_worker(&self) {
622        self.pool.spawned_workers.fetch_sub(1, Ordering::Relaxed);
623    }
624
625    pub(crate) fn register_auxiliary(&self) {
626        self.pool.auxiliary_threads.fetch_add(1, Ordering::Relaxed);
627    }
628
629    pub(crate) fn unregister_auxiliary(&self) {
630        self.pool.auxiliary_threads.fetch_sub(1, Ordering::Relaxed);
631    }
632
633    pub(crate) fn configured_workers(&self) -> usize {
634        self.pool.configured_workers()
635    }
636
637    pub(crate) fn worker_limit(&self) -> usize {
638        self.pool.worker_limit.load(Ordering::Acquire)
639    }
640
641    pub(crate) fn attached_decoders(&self) -> usize {
642        self.pool.attached_decoders.load(Ordering::Acquire)
643    }
644
645    pub(crate) fn limit_epoch(&self) -> usize {
646        self.pool
647            .limit_epoch
648            .load(Ordering::Relaxed)
649            .wrapping_add(self.state.grant_epoch.load(Ordering::Relaxed))
650    }
651
652    pub(crate) fn detach(&self) {
653        if !self.state.attached.swap(false, Ordering::AcqRel) {
654            return;
655        }
656        self.state.queued_tasks.store(0, Ordering::Relaxed);
657        self.state.desired_workers.store(0, Ordering::Relaxed);
658        if self.state.runnable.swap(false, Ordering::AcqRel) {
659            self.pool.runnable_decoders.fetch_sub(1, Ordering::Relaxed);
660        }
661        self.pool.attached_decoders.fetch_sub(1, Ordering::Relaxed);
662        let mut scheduler = self
663            .pool
664            .scheduler
665            .lock()
666            .expect("decoder pool mutex poisoned");
667        let detached = Arc::downgrade(&self.state);
668        scheduler
669            .members
670            .retain(|member| !Weak::ptr_eq(member, &detached));
671        self.pool.rebalance(&mut scheduler, false);
672        drop(scheduler);
673        self.pool.signal.notify_all();
674    }
675}
676
677impl Drop for PoolMember {
678    fn drop(&mut self) {
679        debug_assert_eq!(self.state.busy_workers.load(Ordering::Relaxed), 0);
680        debug_assert_eq!(self.state.waiting_workers.load(Ordering::Relaxed), 0);
681        self.detach();
682    }
683}
684
685pub(crate) struct PoolPermit<'a> {
686    pool: &'a PoolState,
687    member: &'a PoolMemberState,
688}
689
690impl Drop for PoolPermit<'_> {
691    fn drop(&mut self) {
692        self.pool.release(self.member);
693    }
694}
695
696#[cfg(test)]
697mod tests {
698    use super::{DecoderPool, PoolMember};
699    use std::sync::{Arc, Barrier};
700    use std::thread;
701    use std::time::Duration;
702
703
704    /// `release` must not lose the wakeup of a waiter that has checked the
705    /// permit count but not yet parked.
706    ///
707    /// The failure this pins (found as a salmon-side hang, but the bug is
708    /// here): a waiter in `acquire`'s slow path holds the scheduler mutex,
709    /// reads `busy == limit`, and parks. `release` used to decrement
710    /// `busy_workers` (an atomic, outside the mutex) and call `notify_all`
711    /// without the lock -- so the final release could land in the window
712    /// between a waiter's check and its park, notify nobody, and leave the
713    /// pool with free permits, parked waiters, and no future notify. Every
714    /// subsequent acquire then queues behind the dead waiters, because
715    /// `try_acquire_fast` refuses to jump a non-empty queue.
716    ///
717    /// The window is a few instructions wide, so this hammers it: churning
718    /// acquire/release across two members (as two gzip inputs do) through a
719    /// limit of 1 -- the configuration the thread broker's shrinks produce --
720    /// while a resizer toggles the limit, maximising slow-path traffic. Before
721    /// the fix this test hangs within a few hundred thousand iterations; the
722    /// watchdog turns that into a failure rather than a stuck CI job.
723    #[test]
724    fn release_wakes_waiters_that_raced_the_final_release() {
725        const CHURNERS_PER_MEMBER: usize = 1;
726        // Pre-fix this deadlocked at ~700k iterations; 5M gives ~7x margin while
727        // keeping the test under ten seconds in release.
728        const ITERS: u64 = 5_000_000;
729
730        let pool = DecoderPool::builder()
731            .workers(4)
732            .initial_worker_limit(1)
733            .build()
734            .unwrap();
735        let members: Vec<Arc<PoolMember>> = (0..2)
736            .map(|_| Arc::new(pool.state.register_decoder()))
737            .collect();
738
739        let done = Arc::new(std::sync::atomic::AtomicBool::new(false));
740        let progressed = Arc::new(std::sync::atomic::AtomicU64::new(0));
741        let start = Arc::new(Barrier::new(2 * CHURNERS_PER_MEMBER + 1));
742
743        let mut handles = Vec::new();
744        for member in &members {
745            for _ in 0..CHURNERS_PER_MEMBER {
746                let member = Arc::clone(member);
747                let done = Arc::clone(&done);
748                let progressed = Arc::clone(&progressed);
749                let start = Arc::clone(&start);
750                handles.push(thread::spawn(move || {
751                    start.wait();
752                    while !done.load(std::sync::atomic::Ordering::Relaxed) {
753                        // The accounting-region pattern: acquire, tiny region,
754                        // release. Nothing is held across anything blocking.
755                        let permit = member.acquire();
756                        drop(permit);
757                        progressed.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
758                    }
759                }));
760            }
761        }
762        // Deliberately NO limit churn: `set_worker_limit` notifies under the
763        // scheduler lock, which *repairs* a lost wakeup -- a version of this
764        // test with a resizer thread passed against the broken code for
765        // exactly that reason. The hang in the wild appears after the broker
766        // settles and stops moving the limit, leaving nobody to rescue the
767        // parked waiters.
768
769        start.wait();
770        // Watchdog: require continuous global progress. A lost wakeup shows up
771        // as the counter freezing while every churner sits parked in acquire.
772        let deadline = std::time::Instant::now() + Duration::from_secs(60);
773        let mut last = 0_u64;
774        loop {
775            thread::sleep(Duration::from_millis(200));
776            let now = progressed.load(std::sync::atomic::Ordering::Relaxed);
777            if now >= ITERS {
778                break;
779            }
780            assert!(
781                now > last,
782                "no acquire/release progress in 200ms at {now} iterations: \
783                 waiters are parked with free permits -- the release lost their wakeup"
784            );
785            last = now;
786            assert!(
787                std::time::Instant::now() < deadline,
788                "test exceeded its deadline while still progressing; raise ITERS \
789                 only with cause"
790            );
791        }
792        done.store(true, std::sync::atomic::Ordering::Relaxed);
793        for h in handles {
794            h.join().unwrap();
795        }
796    }
797
798    #[test]
799    fn builder_validates_limits() {
800        assert!(DecoderPool::builder().workers(0).build().is_err());
801        assert!(
802            DecoderPool::builder()
803                .workers(4)
804                .initial_worker_limit(5)
805                .build()
806                .is_err()
807        );
808        let pool = DecoderPool::builder()
809            .workers(4)
810            .initial_worker_limit(2)
811            .build()
812            .unwrap();
813        assert_eq!(pool.stats().configured_workers, 4);
814        assert_eq!(pool.stats().worker_limit, 2);
815    }
816
817    #[test]
818    fn shared_limit_bounds_concurrent_execution() {
819        let pool = DecoderPool::builder().workers(2).build().unwrap();
820        let member = Arc::new(pool.state.register_decoder());
821        let start = Arc::new(Barrier::new(5));
822        let maximum = Arc::new(std::sync::atomic::AtomicUsize::new(0));
823        let mut workers = Vec::new();
824        for _ in 0..4 {
825            let member = Arc::clone(&member);
826            let start = Arc::clone(&start);
827            let maximum = Arc::clone(&maximum);
828            workers.push(thread::spawn(move || {
829                start.wait();
830                let _permit = member.acquire();
831                let busy = member.busy_workers();
832                maximum.fetch_max(busy, std::sync::atomic::Ordering::Relaxed);
833                thread::sleep(Duration::from_millis(5));
834            }));
835        }
836        start.wait();
837        for worker in workers {
838            worker.join().unwrap();
839        }
840        assert_eq!(maximum.load(std::sync::atomic::Ordering::Relaxed), 2);
841        assert_eq!(pool.stats().busy_workers, 0);
842    }
843
844    #[test]
845    fn attached_demand_stabilizes_spawn_allowances_until_detach() {
846        let pool = DecoderPool::builder().workers(4).build().unwrap();
847        let first = pool.state.register_decoder();
848        let second = pool.state.register_decoder();
849        first.set_desired_workers(4);
850        second.set_desired_workers(4);
851
852        first.set_queued_tasks(8);
853        assert_eq!(first.granted_workers(), 2);
854        assert_eq!(second.granted_workers(), 2);
855        assert_eq!(pool.stats().runnable_decoders, 1);
856
857        second.set_queued_tasks(8);
858        assert_eq!(first.granted_workers(), 2);
859        assert_eq!(second.granted_workers(), 2);
860        assert_eq!(pool.stats().active_workers, 4);
861
862        let final_second_task = second.acquire();
863        second.set_queued_tasks(0);
864        drop(final_second_task);
865        assert_eq!(first.granted_workers(), 2);
866        assert_eq!(second.granted_workers(), 2);
867        assert_eq!(pool.stats().runnable_decoders, 1);
868
869        drop(second);
870        for queued in 8..(8 + PoolMember::GRANT_GROWTH_QUEUE_EVENTS) {
871            first.set_queued_tasks(queued);
872        }
873        assert_eq!(first.granted_workers(), 4);
874    }
875
876    #[test]
877    fn resizing_redistributes_live_allowances() {
878        let pool = DecoderPool::builder()
879            .workers(4)
880            .initial_worker_limit(2)
881            .build()
882            .unwrap();
883        let first = pool.state.register_decoder();
884        let second = pool.state.register_decoder();
885        first.set_desired_workers(4);
886        second.set_desired_workers(4);
887        first.set_queued_tasks(4);
888        second.set_queued_tasks(4);
889        assert_eq!(pool.stats().active_workers, 2);
890
891        pool.set_worker_limit(4).unwrap();
892        assert_eq!(pool.stats().active_workers, 4);
893        assert_eq!(first.granted_workers(), 2);
894        assert_eq!(second.granted_workers(), 2);
895
896        pool.set_worker_limit(1).unwrap();
897        assert_eq!(pool.stats().active_workers, 1);
898        assert_eq!(first.granted_workers() + second.granted_workers(), 1);
899        assert_eq!(pool.set_worker_limit(0).unwrap_err().requested(), 0);
900    }
901}