use std::{
collections::HashMap,
fmt,
panic::{catch_unwind, AssertUnwindSafe},
sync::{Arc, Mutex},
time::Duration,
};
use tokio::{
sync::{OwnedSemaphorePermit, Semaphore},
time::Instant,
};
use tokio_util::sync::CancellationToken;
use crate::{ModuleHandle, RequestKey, DISPATCH_SATURATION_WAIT, HANDLER_TASK_CAPACITY};
pub(crate) const PERMIT_WAIT_WARNING: Duration = Duration::from_secs(2);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum RequestPool {
Default,
LongHeld,
}
impl RequestPool {
pub fn name(self) -> &'static str {
match self {
Self::Default => "default",
Self::LongHeld => "long_held",
}
}
}
impl fmt::Display for RequestPool {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.name())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct RequestClass {
pub pool: RequestPool,
pub op: Option<&'static str>,
}
impl RequestClass {
pub const fn default_pool() -> Self {
Self {
pool: RequestPool::Default,
op: None,
}
}
pub const fn long_held() -> Self {
Self {
pool: RequestPool::LongHeld,
op: None,
}
}
pub const fn with_op(mut self, op: &'static str) -> Self {
self.op = Some(op);
self
}
}
impl Default for RequestClass {
fn default() -> Self {
Self::default_pool()
}
}
type Classifier = dyn Fn(&[u8]) -> RequestClass + Send + Sync;
#[derive(Clone, Default)]
#[non_exhaustive]
pub struct RequestPools {
long_held: Option<(usize, Arc<Classifier>)>,
}
impl RequestPools {
pub fn new() -> Self {
Self::default()
}
pub fn with_long_held<F>(mut self, limit: usize, classifier: F) -> Self
where
F: Fn(&[u8]) -> RequestClass + Send + Sync + 'static,
{
self.long_held = Some((limit.max(1), Arc::new(classifier)));
self
}
pub fn long_held_limit(&self) -> Option<usize> {
self.long_held.as_ref().map(|(limit, _)| *limit)
}
}
impl fmt::Debug for RequestPools {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RequestPools")
.field("long_held_limit", &self.long_held_limit())
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct RequestPoolSnapshot {
pub pool: RequestPool,
pub in_use: usize,
pub limit: usize,
pub waiting: usize,
pub oldest_wait: Option<Duration>,
}
type WaitingRequests = Arc<Mutex<HashMap<RequestKey, Instant>>>;
#[derive(Clone)]
pub(crate) struct PermitPool {
pub(crate) pool: RequestPool,
pub(crate) limit: usize,
pub(crate) permits: Arc<Semaphore>,
pub(crate) waiting: WaitingRequests,
}
impl PermitPool {
pub(crate) fn new(pool: RequestPool, limit: usize) -> Self {
Self {
pool,
limit,
permits: Arc::new(Semaphore::new(limit)),
waiting: Arc::new(Mutex::new(HashMap::new())),
}
}
fn in_use(&self) -> usize {
self.limit.saturating_sub(self.permits.available_permits())
}
fn lock_waiting(&self) -> std::sync::MutexGuard<'_, HashMap<RequestKey, Instant>> {
self.waiting
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
pub(crate) fn start_waiting(&self, key: RequestKey) -> PermitWait {
let since = Instant::now();
self.lock_waiting().insert(key, since);
PermitWait {
waiting: Arc::clone(&self.waiting),
key,
since,
}
}
pub(crate) fn snapshot(&self) -> RequestPoolSnapshot {
let waiting = self.lock_waiting();
let oldest_wait = waiting.values().min().map(Instant::elapsed);
let count = waiting.len();
drop(waiting);
RequestPoolSnapshot {
pool: self.pool,
in_use: self.in_use(),
limit: self.limit,
waiting: count,
oldest_wait,
}
}
fn saturation(&self, named: bool) -> Option<String> {
let snapshot = self.snapshot();
let age = snapshot.oldest_wait.filter(|age| {
self.permits.available_permits() == 0 && *age > DISPATCH_SATURATION_WAIT
})?;
let pool = if named {
format!(" in the {} pool", self.pool.name())
} else {
String::new()
};
Some(format!(
"request dispatch saturated{pool}: {}/{} in use, oldest waiting {:.1} s",
snapshot.in_use,
snapshot.limit,
age.as_secs_f64(),
))
}
}
pub(crate) struct PermitWait {
waiting: WaitingRequests,
key: RequestKey,
since: Instant,
}
impl Drop for PermitWait {
fn drop(&mut self) {
self.waiting
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&self.key);
}
}
pub(crate) struct DispatchPools {
pub(crate) default: PermitPool,
pub(crate) long_held: Option<(PermitPool, Arc<Classifier>)>,
}
impl DispatchPools {
pub(crate) fn new(config: RequestPools) -> Self {
Self::with_default_limit(HANDLER_TASK_CAPACITY, config)
}
pub(crate) fn with_default_limit(default_limit: usize, config: RequestPools) -> Self {
Self {
default: PermitPool::new(RequestPool::Default, default_limit),
long_held: config.long_held.map(|(limit, classifier)| {
(PermitPool::new(RequestPool::LongHeld, limit), classifier)
}),
}
}
pub(crate) fn pools(&self) -> impl Iterator<Item = &PermitPool> {
std::iter::once(&self.default).chain(self.long_held.as_ref().map(|(pool, _)| pool))
}
pub(crate) fn classify(&self, body: &[u8]) -> (&PermitPool, Option<&'static str>) {
let Some((long_held, classifier)) = &self.long_held else {
return (&self.default, None);
};
let class = match catch_unwind(AssertUnwindSafe(|| classifier(body))) {
Ok(class) => class,
Err(_) => {
tracing::warn!(
"the module's request classifier panicked; the request uses the default pool"
);
RequestClass::default_pool()
}
};
let pool = match class.pool {
RequestPool::LongHeld => long_held,
RequestPool::Default => &self.default,
};
(pool, class.op)
}
pub(crate) fn saturation(&self) -> Option<String> {
let named = self.long_held.is_some();
let sentences = self
.pools()
.filter_map(|pool| pool.saturation(named))
.collect::<Vec<_>>();
(!sentences.is_empty()).then(|| sentences.join("; "))
}
pub(crate) fn snapshots(&self) -> Vec<RequestPoolSnapshot> {
self.pools().map(PermitPool::snapshot).collect()
}
pub(crate) fn close(&self) {
for pool in self.pools() {
pool.permits.close();
}
}
}
pub(crate) enum PermitWaitOutcome {
Acquired(OwnedSemaphorePermit),
Cancelled,
Closed,
}
pub(crate) struct WaitingRequest {
pub(crate) op: Option<&'static str>,
pub(crate) channel: u16,
pub(crate) corr: u64,
}
pub(crate) async fn wait_for_permit(
pool: &PermitPool,
wait: PermitWait,
request: WaitingRequest,
cancelled: &CancellationToken,
) -> PermitWaitOutcome {
let acquire = Arc::clone(&pool.permits).acquire_owned();
let warn_at = tokio::time::sleep_until(wait.since + PERMIT_WAIT_WARNING);
tokio::pin!(acquire, warn_at);
let mut warned = false;
let outcome = loop {
tokio::select! {
biased;
permit = &mut acquire => {
break match permit {
Ok(permit) => PermitWaitOutcome::Acquired(permit),
Err(_) => PermitWaitOutcome::Closed,
};
}
() = cancelled.cancelled() => break PermitWaitOutcome::Cancelled,
() = &mut warn_at, if !warned => {
warned = true;
tracing::warn!(
op = request.op.unwrap_or("<none>"),
pool = pool.pool.name(),
in_use = pool.in_use(),
limit = pool.limit,
channel = request.channel,
corr = request.corr,
waited_ms = u64::try_from(wait.since.elapsed().as_millis()).unwrap_or(u64::MAX),
"data request is still waiting for a handler slot after {} s",
PERMIT_WAIT_WARNING.as_secs(),
);
}
}
};
drop(wait);
outcome
}
impl ModuleHandle {
pub fn request_pool_snapshots(&self) -> Vec<RequestPoolSnapshot> {
self.dispatch_pools().snapshots()
}
pub(crate) fn install_dispatch_pools(&self, config: RequestPools) {
*self
.shared
.dispatch_pools
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) =
Arc::new(DispatchPools::new(config));
}
pub(crate) fn dispatch_pools(&self) -> Arc<DispatchPools> {
Arc::clone(
&self
.shared
.dispatch_pools
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()),
)
}
}