Skip to main content

subc_client_rs/
request_pools.rs

1//! Handler-slot pools for data requests.
2//!
3//! Every data request waits for a handler slot before its handler runs, and
4//! keeps the slot until the handler returns. By default all requests share one
5//! pool of 64 slots. A module whose requests are sometimes held for minutes can
6//! give those requests a pool of their own, so they can never take every slot
7//! the module's short requests need. The module declares that pool, and a
8//! classifier that sorts requests into pools, with
9//! [`ModuleHandler::request_pools`](crate::ModuleHandler::request_pools).
10
11use std::{
12    collections::HashMap,
13    fmt,
14    panic::{catch_unwind, AssertUnwindSafe},
15    sync::{Arc, Mutex},
16    time::Duration,
17};
18
19use tokio::{
20    sync::{OwnedSemaphorePermit, Semaphore},
21    time::Instant,
22};
23use tokio_util::sync::CancellationToken;
24
25use crate::{ModuleHandle, RequestKey, DISPATCH_SATURATION_WAIT, HANDLER_TASK_CAPACITY};
26
27/// How long a data request waits for a handler slot before the SDK logs a
28/// warning about it. The warning is logged once, while the request is still
29/// waiting.
30pub(crate) const PERMIT_WAIT_WARNING: Duration = Duration::from_secs(2);
31
32/// The handler-slot pool a data request waits in.
33#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
34#[non_exhaustive]
35pub enum RequestPool {
36    /// The pool every data request uses unless the module's classifier puts
37    /// it elsewhere. It has 64 slots.
38    Default,
39    /// The pool a module reserves for requests that can be held for a long
40    /// time, with the limit the module sets in [`RequestPools::with_long_held`].
41    LongHeld,
42}
43
44impl RequestPool {
45    /// The name the SDK uses for this pool in logs and health details.
46    pub fn name(self) -> &'static str {
47        match self {
48            Self::Default => "default",
49            Self::LongHeld => "long_held",
50        }
51    }
52}
53
54impl fmt::Display for RequestPool {
55    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
56        f.write_str(self.name())
57    }
58}
59
60/// A classifier's answer about one data request: the pool it waits in, and
61/// an optional label naming the operation, used only in the SDK's logs.
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63#[non_exhaustive]
64pub struct RequestClass {
65    pub pool: RequestPool,
66    /// A fixed name for the operation, such as `"research.run"`. It is
67    /// `'static` so a label can never carry text taken from the request body.
68    pub op: Option<&'static str>,
69}
70
71impl RequestClass {
72    /// A request for the default pool, with no operation label.
73    pub const fn default_pool() -> Self {
74        Self {
75            pool: RequestPool::Default,
76            op: None,
77        }
78    }
79
80    /// A request for the long-held pool, with no operation label.
81    pub const fn long_held() -> Self {
82        Self {
83            pool: RequestPool::LongHeld,
84            op: None,
85        }
86    }
87
88    /// The same class, labelled with the operation name `op`.
89    pub const fn with_op(mut self, op: &'static str) -> Self {
90        self.op = Some(op);
91        self
92    }
93}
94
95impl Default for RequestClass {
96    fn default() -> Self {
97        Self::default_pool()
98    }
99}
100
101type Classifier = dyn Fn(&[u8]) -> RequestClass + Send + Sync;
102
103/// How a module's data requests share handler slots, returned from
104/// [`ModuleHandler::request_pools`](crate::ModuleHandler::request_pools).
105///
106/// [`RequestPools::new`] is the behaviour a module gets when it declares
107/// nothing: one pool of 64 slots for every data request.
108/// [`RequestPools::with_long_held`] adds a second pool and the classifier
109/// that decides which requests use it.
110#[derive(Clone, Default)]
111#[non_exhaustive]
112pub struct RequestPools {
113    long_held: Option<(usize, Arc<Classifier>)>,
114}
115
116impl RequestPools {
117    /// One pool of 64 slots for every data request, with no classifier.
118    pub fn new() -> Self {
119        Self::default()
120    }
121
122    /// Add a long-held pool of `limit` slots, and the classifier that puts
123    /// each data request in a pool.
124    ///
125    /// The SDK calls `classifier` with the request's raw body, before the
126    /// request takes a slot; the SDK itself never interprets a body. A
127    /// request the classifier puts in [`RequestPool::LongHeld`] waits only
128    /// for a long-held slot, and every other request waits only for a
129    /// default slot, so a full pool never delays requests of the other one.
130    ///
131    /// The classifier runs on the task that reads the connection's frames,
132    /// so it must be cheap and must not block: peek at the operation name,
133    /// do not decode the whole body. If it panics, the request uses the
134    /// default pool. A `limit` of 0 is raised to 1, so long-held requests
135    /// are delayed, never stranded.
136    pub fn with_long_held<F>(mut self, limit: usize, classifier: F) -> Self
137    where
138        F: Fn(&[u8]) -> RequestClass + Send + Sync + 'static,
139    {
140        self.long_held = Some((limit.max(1), Arc::new(classifier)));
141        self
142    }
143
144    /// The long-held pool's slot limit, or `None` when there is no long-held
145    /// pool.
146    pub fn long_held_limit(&self) -> Option<usize> {
147        self.long_held.as_ref().map(|(limit, _)| *limit)
148    }
149}
150
151impl fmt::Debug for RequestPools {
152    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
153        f.debug_struct("RequestPools")
154            .field("long_held_limit", &self.long_held_limit())
155            .finish_non_exhaustive()
156    }
157}
158
159/// Figures for one handler-slot pool at one moment, from
160/// [`ModuleHandle::request_pool_snapshots`].
161#[derive(Debug, Clone, PartialEq, Eq)]
162#[non_exhaustive]
163pub struct RequestPoolSnapshot {
164    pub pool: RequestPool,
165    /// Slots held by requests whose handlers are running.
166    pub in_use: usize,
167    /// The pool's number of slots.
168    pub limit: usize,
169    /// Requests received and waiting for a slot in this pool.
170    pub waiting: usize,
171    /// How long the request that has waited longest has been waiting, or
172    /// `None` when no request is waiting.
173    pub oldest_wait: Option<Duration>,
174}
175
176type WaitingRequests = Arc<Mutex<HashMap<RequestKey, Instant>>>;
177
178/// One pool's slots and the requests waiting for them, shared by the
179/// connection's dispatcher, its request tasks and its [`ModuleHandle`].
180#[derive(Clone)]
181pub(crate) struct PermitPool {
182    pub(crate) pool: RequestPool,
183    pub(crate) limit: usize,
184    pub(crate) permits: Arc<Semaphore>,
185    /// When each request in this pool started waiting for a slot. Entries are
186    /// added before the request task is spawned and removed once it has a slot
187    /// or has stopped waiting. The lock is never held over an await or handler
188    /// work, so health and snapshots can read it without queueing behind the
189    /// work they measure.
190    pub(crate) waiting: WaitingRequests,
191}
192
193impl PermitPool {
194    pub(crate) fn new(pool: RequestPool, limit: usize) -> Self {
195        Self {
196            pool,
197            limit,
198            permits: Arc::new(Semaphore::new(limit)),
199            waiting: Arc::new(Mutex::new(HashMap::new())),
200        }
201    }
202
203    fn in_use(&self) -> usize {
204        self.limit.saturating_sub(self.permits.available_permits())
205    }
206
207    fn lock_waiting(&self) -> std::sync::MutexGuard<'_, HashMap<RequestKey, Instant>> {
208        self.waiting
209            .lock()
210            .unwrap_or_else(|poisoned| poisoned.into_inner())
211    }
212
213    /// Record that the request `key` has started waiting for a slot in this
214    /// pool. The record is removed when the returned guard drops.
215    pub(crate) fn start_waiting(&self, key: RequestKey) -> PermitWait {
216        let since = Instant::now();
217        self.lock_waiting().insert(key, since);
218        PermitWait {
219            waiting: Arc::clone(&self.waiting),
220            key,
221            since,
222        }
223    }
224
225    pub(crate) fn snapshot(&self) -> RequestPoolSnapshot {
226        let waiting = self.lock_waiting();
227        let oldest_wait = waiting.values().min().map(Instant::elapsed);
228        let count = waiting.len();
229        drop(waiting);
230        RequestPoolSnapshot {
231            pool: self.pool,
232            in_use: self.in_use(),
233            limit: self.limit,
234            waiting: count,
235            oldest_wait,
236        }
237    }
238
239    /// The saturation sentence for health, when every slot is in use and a
240    /// request has waited longer than [`DISPATCH_SATURATION_WAIT`] for one.
241    /// `named` adds the pool's name, which a module with a single pool never
242    /// gets, so its wording is unchanged from before pools existed.
243    fn saturation(&self, named: bool) -> Option<String> {
244        let snapshot = self.snapshot();
245        let age = snapshot.oldest_wait.filter(|age| {
246            self.permits.available_permits() == 0 && *age > DISPATCH_SATURATION_WAIT
247        })?;
248        let pool = if named {
249            format!(" in the {} pool", self.pool.name())
250        } else {
251            String::new()
252        };
253        Some(format!(
254            "request dispatch saturated{pool}: {}/{} in use, oldest waiting {:.1} s",
255            snapshot.in_use,
256            snapshot.limit,
257            age.as_secs_f64(),
258        ))
259    }
260}
261
262/// Holds a request's entry in its pool's `waiting` map and removes it on drop,
263/// so the entry goes even if the waiting task exits on connection close or
264/// panics.
265pub(crate) struct PermitWait {
266    waiting: WaitingRequests,
267    key: RequestKey,
268    since: Instant,
269}
270
271impl Drop for PermitWait {
272    fn drop(&mut self) {
273        self.waiting
274            .lock()
275            .unwrap_or_else(|poisoned| poisoned.into_inner())
276            .remove(&self.key);
277    }
278}
279
280/// The pools of one connection and the module's classifier.
281pub(crate) struct DispatchPools {
282    pub(crate) default: PermitPool,
283    /// Present exactly when the module declared a classifier.
284    pub(crate) long_held: Option<(PermitPool, Arc<Classifier>)>,
285}
286
287impl DispatchPools {
288    pub(crate) fn new(config: RequestPools) -> Self {
289        Self::with_default_limit(HANDLER_TASK_CAPACITY, config)
290    }
291
292    pub(crate) fn with_default_limit(default_limit: usize, config: RequestPools) -> Self {
293        Self {
294            default: PermitPool::new(RequestPool::Default, default_limit),
295            long_held: config.long_held.map(|(limit, classifier)| {
296                (PermitPool::new(RequestPool::LongHeld, limit), classifier)
297            }),
298        }
299    }
300
301    /// Every pool, the default pool first.
302    pub(crate) fn pools(&self) -> impl Iterator<Item = &PermitPool> {
303        std::iter::once(&self.default).chain(self.long_held.as_ref().map(|(pool, _)| pool))
304    }
305
306    /// The pool a request with `body` waits in, and its operation label. With
307    /// no classifier every request uses the default pool and has no label.
308    pub(crate) fn classify(&self, body: &[u8]) -> (&PermitPool, Option<&'static str>) {
309        let Some((long_held, classifier)) = &self.long_held else {
310            return (&self.default, None);
311        };
312        let class = match catch_unwind(AssertUnwindSafe(|| classifier(body))) {
313            Ok(class) => class,
314            Err(_) => {
315                tracing::warn!(
316                    "the module's request classifier panicked; the request uses the default pool"
317                );
318                RequestClass::default_pool()
319            }
320        };
321        let pool = match class.pool {
322            RequestPool::LongHeld => long_held,
323            RequestPool::Default => &self.default,
324        };
325        (pool, class.op)
326    }
327
328    /// The saturation sentences of every saturated pool, joined, for health.
329    /// A module with one pool gets the same wording as before pools existed;
330    /// with two, each sentence names its pool.
331    pub(crate) fn saturation(&self) -> Option<String> {
332        let named = self.long_held.is_some();
333        let sentences = self
334            .pools()
335            .filter_map(|pool| pool.saturation(named))
336            .collect::<Vec<_>>();
337        (!sentences.is_empty()).then(|| sentences.join("; "))
338    }
339
340    pub(crate) fn snapshots(&self) -> Vec<RequestPoolSnapshot> {
341        self.pools().map(PermitPool::snapshot).collect()
342    }
343
344    /// Close every pool, so requests still waiting for a slot stop waiting.
345    pub(crate) fn close(&self) {
346        for pool in self.pools() {
347            pool.permits.close();
348        }
349    }
350}
351
352/// How a request's wait for a handler slot ended.
353pub(crate) enum PermitWaitOutcome {
354    Acquired(OwnedSemaphorePermit),
355    /// The request was cancelled before it got a slot.
356    Cancelled,
357    /// The pool was closed because the connection ended.
358    Closed,
359}
360
361/// Where a waiting request came from, for the slow-wait warning.
362pub(crate) struct WaitingRequest {
363    pub(crate) op: Option<&'static str>,
364    pub(crate) channel: u16,
365    pub(crate) corr: u64,
366}
367
368/// Wait for a slot in `pool`, stopping at once if `cancelled` fires. Once the
369/// request has waited [`PERMIT_WAIT_WARNING`] and is still waiting, log one
370/// warning naming its operation label, pool, the pool's figures, and its route
371/// channel and correlation id. The request body is never logged.
372pub(crate) async fn wait_for_permit(
373    pool: &PermitPool,
374    wait: PermitWait,
375    request: WaitingRequest,
376    cancelled: &CancellationToken,
377) -> PermitWaitOutcome {
378    let acquire = Arc::clone(&pool.permits).acquire_owned();
379    let warn_at = tokio::time::sleep_until(wait.since + PERMIT_WAIT_WARNING);
380    tokio::pin!(acquire, warn_at);
381    let mut warned = false;
382    let outcome = loop {
383        tokio::select! {
384            // A slot is taken in preference to the other two. Connection
385            // teardown closes the pools before it cancels requests, so a
386            // waiting request then sees the closed pool and ends without
387            // answering, as it did before cancellation was watched here.
388            biased;
389            permit = &mut acquire => {
390                break match permit {
391                    Ok(permit) => PermitWaitOutcome::Acquired(permit),
392                    Err(_) => PermitWaitOutcome::Closed,
393                };
394            }
395            () = cancelled.cancelled() => break PermitWaitOutcome::Cancelled,
396            // A finished sleep stays ready, so `warned` is what limits the
397            // warning to once.
398            () = &mut warn_at, if !warned => {
399                warned = true;
400                tracing::warn!(
401                    op = request.op.unwrap_or("<none>"),
402                    pool = pool.pool.name(),
403                    in_use = pool.in_use(),
404                    limit = pool.limit,
405                    channel = request.channel,
406                    corr = request.corr,
407                    waited_ms = u64::try_from(wait.since.elapsed().as_millis()).unwrap_or(u64::MAX),
408                    "data request is still waiting for a handler slot after {} s",
409                    PERMIT_WAIT_WARNING.as_secs(),
410                );
411            }
412        }
413    };
414    drop(wait);
415    outcome
416}
417
418impl ModuleHandle {
419    /// Figures for each handler-slot pool of this connection, the default
420    /// pool first, then the long-held pool when the module declared one in
421    /// [`ModuleHandler::request_pools`](crate::ModuleHandler::request_pools).
422    pub fn request_pool_snapshots(&self) -> Vec<RequestPoolSnapshot> {
423        self.dispatch_pools().snapshots()
424    }
425
426    /// Replace this connection's pools with the ones `config` declares.
427    /// Called once, before the connection's dispatcher exists.
428    pub(crate) fn install_dispatch_pools(&self, config: RequestPools) {
429        *self
430            .shared
431            .dispatch_pools
432            .lock()
433            .unwrap_or_else(|poisoned| poisoned.into_inner()) =
434            Arc::new(DispatchPools::new(config));
435    }
436
437    pub(crate) fn dispatch_pools(&self) -> Arc<DispatchPools> {
438        Arc::clone(
439            &self
440                .shared
441                .dispatch_pools
442                .lock()
443                .unwrap_or_else(|poisoned| poisoned.into_inner()),
444        )
445    }
446}