use std::collections::VecDeque;
use std::fmt;
use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use futures::FutureExt;
use tokio::sync::{Mutex, Notify, OwnedSemaphorePermit, Semaphore};
use tokio_util::sync::CancellationToken;
use tracing::{debug, trace, warn};
#[async_trait::async_trait]
pub trait DhtTask: Send + fmt::Debug {
async fn run(self: Box<Self>);
fn name(&self) -> &'static str;
}
pub type BoxedDhtTask = Box<dyn DhtTask>;
pub const DEFAULT_NUM_CONCURRENT: usize = 15;
pub struct DhtTaskExecutor {
inner: Arc<Mutex<DhtTaskExecutorInner>>,
semaphore: Arc<Semaphore>,
num_concurrent: usize,
shutdown: CancellationToken,
idle_notify: Arc<Notify>,
}
struct DhtTaskExecutorInner {
queue: VecDeque<BoxedDhtTask>,
executing: usize,
peak_queue_size: usize,
}
impl DhtTaskExecutor {
pub fn new(num_concurrent: usize) -> Self {
let num_concurrent = num_concurrent.max(1);
Self {
inner: Arc::new(Mutex::new(DhtTaskExecutorInner {
queue: VecDeque::new(),
executing: 0,
peak_queue_size: 0,
})),
semaphore: Arc::new(Semaphore::new(num_concurrent)),
num_concurrent,
shutdown: CancellationToken::new(),
idle_notify: Arc::new(Notify::new()),
}
}
pub async fn add_task(&self, task: BoxedDhtTask) -> bool {
if self.shutdown.is_cancelled() {
return false;
}
let task_name = task.name();
let mut inner = self.inner.lock().await;
if self.shutdown.is_cancelled() {
return false;
}
inner.queue.push_back(task);
inner.peak_queue_size = inner.peak_queue_size.max(inner.queue.len());
trace!(
task = task_name,
queue_len = inner.queue.len(),
executing = inner.executing,
"DHT task enqueued"
);
drop(inner);
self.dispatch_pending().await;
true
}
pub async fn try_add_task_if_idle(&self, task: BoxedDhtTask) -> bool {
if self.shutdown.is_cancelled() {
return false;
}
let task_name = task.name();
let mut inner = self.inner.lock().await;
if self.shutdown.is_cancelled() {
return false;
}
if inner.executing != 0 || !inner.queue.is_empty() {
return false;
}
inner.queue.push_back(task);
inner.peak_queue_size = inner.peak_queue_size.max(inner.queue.len());
trace!(task = task_name, "DHT idle periodic task enqueued");
drop(inner);
self.dispatch_pending().await;
true
}
pub async fn executing_count(&self) -> usize {
self.inner.lock().await.executing
}
pub async fn queue_size(&self) -> usize {
self.inner.lock().await.queue.len()
}
pub async fn peak_queue_size(&self) -> usize {
self.inner.lock().await.peak_queue_size
}
async fn dispatch_pending(&self) {
if self.shutdown.is_cancelled() {
return;
}
let sem = Arc::clone(&self.semaphore);
loop {
let task = {
let mut inner = self.inner.lock().await;
if inner.queue.is_empty() {
break;
}
inner.queue.pop_front()
};
let Some(task) = task else { break };
let task_name = task.name();
match sem.clone().try_acquire_owned() {
Ok(permit) => {
let mut inner = self.inner.lock().await;
inner.executing += 1;
drop(inner);
debug!(task = task_name, "DHT task dispatched");
self.spawn_task(task, permit);
}
Err(_) => {
let mut inner = self.inner.lock().await;
if !self.shutdown.is_cancelled() {
inner.queue.push_front(task);
}
break;
}
}
}
}
async fn run_task(task: BoxedDhtTask, shutdown: &CancellationToken) {
let task_name = task.name();
let result = AssertUnwindSafe(async {
tokio::select! {
_ = shutdown.cancelled() => {}
_ = task.run() => {}
}
})
.catch_unwind()
.await;
if result.is_err() {
warn!(task = task_name, "DHT task panicked; executor continues");
}
}
fn spawn_task(&self, task: BoxedDhtTask, _permit: OwnedSemaphorePermit) {
let inner = Arc::clone(&self.inner);
let semaphore = Arc::clone(&self.semaphore);
let shutdown = self.shutdown.clone();
let idle_notify = Arc::clone(&self.idle_notify);
let run_and_redispatch = async move {
let _held = _permit;
Self::run_task(task, &shutdown).await;
{
let mut guard = inner.lock().await;
guard.executing = guard.executing.saturating_sub(1);
}
idle_notify.notify_one();
drop(_held);
loop {
if shutdown.is_cancelled() {
let mut guard = inner.lock().await;
guard.queue.clear();
break;
}
let next_task = {
let mut guard = inner.lock().await;
guard.queue.pop_front()
};
let Some(next_task) = next_task else { break };
match semaphore.clone().try_acquire_owned() {
Ok(permit) => {
{
let mut guard = inner.lock().await;
guard.executing += 1;
}
debug!(
task = next_task.name(),
"DHT task dispatched (after completion)"
);
let _held2 = permit;
Self::run_task(next_task, &shutdown).await;
{
let mut guard = inner.lock().await;
guard.executing = guard.executing.saturating_sub(1);
}
idle_notify.notify_one();
drop(_held2);
}
Err(_) => {
let mut guard = inner.lock().await;
if !shutdown.is_cancelled() {
guard.queue.push_front(next_task);
}
break;
}
}
}
};
tokio::spawn(run_and_redispatch);
}
pub async fn shutdown(&self) {
self.cancel();
self.inner.lock().await.queue.clear();
loop {
let notified = self.idle_notify.notified();
if self.executing_count().await == 0 {
break;
}
notified.await;
}
}
pub fn cancel(&self) {
self.shutdown.cancel();
}
}
impl fmt::Debug for DhtTaskExecutor {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("DhtTaskExecutor")
.field("num_concurrent", &self.num_concurrent)
.finish()
}
}
pub struct DhtTaskQueue {
periodic_executor_1: DhtTaskExecutor,
periodic_executor_2: DhtTaskExecutor,
immediate_executor: DhtTaskExecutor,
}
impl DhtTaskQueue {
pub fn new() -> Self {
Self::with_concurrency(DEFAULT_NUM_CONCURRENT)
}
pub fn with_concurrency(num_concurrent: usize) -> Self {
Self {
periodic_executor_1: DhtTaskExecutor::new(num_concurrent),
periodic_executor_2: DhtTaskExecutor::new(num_concurrent),
immediate_executor: DhtTaskExecutor::new(num_concurrent),
}
}
pub async fn add_periodic_task_1(&self, task: BoxedDhtTask) -> bool {
self.periodic_executor_1.add_task(task).await
}
pub async fn try_add_periodic_task_1_if_idle(&self, task: BoxedDhtTask) -> bool {
self.periodic_executor_1.try_add_task_if_idle(task).await
}
pub async fn add_periodic_task_2(&self, task: BoxedDhtTask) -> bool {
self.periodic_executor_2.add_task(task).await
}
pub async fn try_add_periodic_task_2_if_idle(&self, task: BoxedDhtTask) -> bool {
self.periodic_executor_2.try_add_task_if_idle(task).await
}
pub async fn add_immediate_task(&self, task: BoxedDhtTask) -> bool {
self.immediate_executor.add_task(task).await
}
pub fn periodic_executor_1(&self) -> &DhtTaskExecutor {
&self.periodic_executor_1
}
pub fn periodic_executor_2(&self) -> &DhtTaskExecutor {
&self.periodic_executor_2
}
pub fn immediate_executor(&self) -> &DhtTaskExecutor {
&self.immediate_executor
}
pub async fn shutdown(&self) {
tokio::join!(
self.periodic_executor_1.shutdown(),
self.periodic_executor_2.shutdown(),
self.immediate_executor.shutdown(),
);
}
pub fn cancel(&self) {
self.periodic_executor_1.cancel();
self.periodic_executor_2.cancel();
self.immediate_executor.cancel();
}
}
impl Default for DhtTaskQueue {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for DhtTaskQueue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("DhtTaskQueue").finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Debug)]
struct CountTask {
counter: Arc<AtomicUsize>,
name: &'static str,
}
#[async_trait::async_trait]
impl DhtTask for CountTask {
async fn run(self: Box<Self>) {
self.counter.fetch_add(1, Ordering::SeqCst);
}
fn name(&self) -> &'static str {
self.name
}
}
#[tokio::test]
async fn test_executor_dispatches_task() {
let executor = DhtTaskExecutor::new(2);
let counter = Arc::new(AtomicUsize::new(0));
executor
.add_task(Box::new(CountTask {
counter: Arc::clone(&counter),
name: "test",
}))
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_executor_concurrency_limit() {
let executor = DhtTaskExecutor::new(2);
let counter = Arc::new(AtomicUsize::new(0));
for _ in 0..5 {
executor
.add_task(Box::new(CountTask {
counter: Arc::clone(&counter),
name: "concurrent-test",
}))
.await;
}
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
assert_eq!(counter.load(Ordering::SeqCst), 5);
assert_eq!(executor.executing_count().await, 0);
}
#[tokio::test]
async fn test_task_queue_three_lanes() {
let queue = DhtTaskQueue::new();
let c1 = Arc::new(AtomicUsize::new(0));
let c2 = Arc::new(AtomicUsize::new(0));
let c3 = Arc::new(AtomicUsize::new(0));
queue
.add_periodic_task_1(Box::new(CountTask {
counter: Arc::clone(&c1),
name: "p1",
}))
.await;
queue
.add_periodic_task_2(Box::new(CountTask {
counter: Arc::clone(&c2),
name: "p2",
}))
.await;
queue
.add_immediate_task(Box::new(CountTask {
counter: Arc::clone(&c3),
name: "imm",
}))
.await;
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert_eq!(c1.load(Ordering::SeqCst), 1);
assert_eq!(c2.load(Ordering::SeqCst), 1);
assert_eq!(c3.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_executor_queue_size() {
let executor = DhtTaskExecutor::new(1);
let counter = Arc::new(AtomicUsize::new(0));
#[derive(Debug)]
struct SlowTask {
counter: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl DhtTask for SlowTask {
async fn run(self: Box<Self>) {
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
self.counter.fetch_add(1, Ordering::SeqCst);
}
fn name(&self) -> &'static str {
"slow"
}
}
executor
.add_task(Box::new(SlowTask {
counter: Arc::clone(&counter),
}))
.await;
for _ in 0..3 {
executor
.add_task(Box::new(CountTask {
counter: Arc::clone(&counter),
name: "queued",
}))
.await;
}
let executing = executor.executing_count().await;
let queued = executor.queue_size().await;
assert!(
executing + queued >= 3,
"executing={}, queued={}",
executing,
queued
);
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
assert_eq!(counter.load(Ordering::SeqCst), 4);
}
#[tokio::test]
async fn test_executor_coalesces_periodic_work_while_busy() {
let executor = DhtTaskExecutor::new(1);
let started = Arc::new(tokio::sync::Notify::new());
let release = Arc::new(tokio::sync::Notify::new());
let counter = Arc::new(AtomicUsize::new(0));
#[derive(Debug)]
struct BlockingTask {
started: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
counter: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl DhtTask for BlockingTask {
async fn run(self: Box<Self>) {
self.counter.fetch_add(1, Ordering::SeqCst);
self.started.notify_one();
self.release.notified().await;
}
fn name(&self) -> &'static str {
"blocking"
}
}
executor
.add_task(Box::new(BlockingTask {
started: Arc::clone(&started),
release: Arc::clone(&release),
counter: Arc::clone(&counter),
}))
.await;
started.notified().await;
assert!(
!executor
.try_add_task_if_idle(Box::new(CountTask {
counter: Arc::clone(&counter),
name: "coalesced",
}))
.await
);
assert_eq!(executor.queue_size().await, 0);
release.notify_one();
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_executor_shutdown_cancels_running_task() {
let executor = DhtTaskExecutor::new(1);
let started = Arc::new(tokio::sync::Notify::new());
let counter = Arc::new(AtomicUsize::new(0));
#[derive(Debug)]
struct NeverEndingTask {
started: Arc<tokio::sync::Notify>,
counter: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl DhtTask for NeverEndingTask {
async fn run(self: Box<Self>) {
self.counter.fetch_add(1, Ordering::SeqCst);
self.started.notify_one();
std::future::pending::<()>().await;
}
fn name(&self) -> &'static str {
"never-ending"
}
}
executor
.add_task(Box::new(NeverEndingTask {
started: Arc::clone(&started),
counter: Arc::clone(&counter),
}))
.await;
started.notified().await;
executor.shutdown().await;
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
assert_eq!(executor.executing_count().await, 0);
assert!(
!executor
.try_add_task_if_idle(Box::new(CountTask {
counter: Arc::clone(&counter),
name: "after-shutdown",
}))
.await
);
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_executor_does_not_stall_after_task_panic() {
#[derive(Debug)]
struct PanicTask {
started: Arc<tokio::sync::Notify>,
}
#[async_trait::async_trait]
impl DhtTask for PanicTask {
async fn run(self: Box<Self>) {
self.started.notify_one();
panic!("intentional task panic");
}
fn name(&self) -> &'static str {
"panic"
}
}
let executor = DhtTaskExecutor::new(1);
let started = Arc::new(tokio::sync::Notify::new());
assert!(
executor
.add_task(Box::new(PanicTask {
started: Arc::clone(&started),
}))
.await
);
started.notified().await;
let shutdown =
tokio::time::timeout(std::time::Duration::from_millis(500), executor.shutdown()).await;
assert!(shutdown.is_ok(), "executor shutdown should not stall");
assert_eq!(executor.executing_count().await, 0);
}
}