use crate::{
telemetry::metrics::raw::Gauge,
utils::{extract_panic_message, supervision::Tree},
Error,
};
use commonware_utils::{
channel::oneshot,
sync::{Mutex, Once},
};
use futures::{
future::{select, Either},
pin_mut,
stream::{AbortHandle, Abortable, Aborted},
FutureExt as _,
};
use std::{
any::Any,
future::Future,
panic::{resume_unwind, AssertUnwindSafe},
pin::Pin,
sync::Arc,
task::{Context, Poll},
};
use tracing::error;
pub struct Handle<T>
where
T: Send + 'static,
{
state: HandleState<T>,
}
enum HandleState<T>
where
T: Send + 'static,
{
Task {
receiver: oneshot::Receiver<Result<T, Error>>,
abort_handle: AbortHandle,
metric: MetricHandle,
},
Completion {
future: Abortable<Completion<T>>,
abort_handle: AbortHandle,
},
}
enum Completion<T>
where
T: Send + 'static,
{
Receiver(oneshot::Receiver<Result<T, Error>>),
Future(Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'static>>),
}
impl<T> Unpin for Completion<T> where T: Send + 'static {}
impl<T> Future for Completion<T>
where
T: Send + 'static,
{
type Output = Result<T, Error>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match &mut *self {
Self::Receiver(receiver) => Pin::new(receiver)
.poll(cx)
.map(|result| result.unwrap_or(Err(Error::Closed))),
Self::Future(future) => future.as_mut().poll(cx),
}
}
}
impl<T> Handle<T>
where
T: Send + 'static,
{
#[inline(always)]
pub(crate) fn init<F>(
f: F,
metric: MetricHandle,
panicker: Panicker,
tree: Arc<Tree>,
) -> (impl Future<Output = ()>, Self)
where
F: Future<Output = T> + Send + 'static,
{
let (sender, receiver) = oneshot::channel();
let (abort_handle, abort_registration) = AbortHandle::new_pair();
let metric_handle = metric.clone();
let task = async move {
let result =
Abortable::new(AssertUnwindSafe(f).catch_unwind(), abort_registration).await;
match result {
Ok(Ok(result)) => {
let _ = sender.send(Ok(result));
}
Ok(Err(panic)) => {
panicker.notify(panic);
let _ = sender.send(Err(Error::Exited));
}
Err(Aborted) => {}
}
tree.abort();
metric_handle.finish();
};
(
task,
Self {
state: HandleState::Task {
receiver,
abort_handle,
metric,
},
},
)
}
pub fn from_receiver(receiver: oneshot::Receiver<Result<T, Error>>) -> Self {
let (abort_handle, abort_registration) = AbortHandle::new_pair();
Self {
state: HandleState::Completion {
future: Abortable::new(Completion::Receiver(receiver), abort_registration),
abort_handle,
},
}
}
pub fn from_future<F>(future: F) -> Self
where
F: Future<Output = Result<T, Error>> + Send + 'static,
{
let (abort_handle, abort_registration) = AbortHandle::new_pair();
Self {
state: HandleState::Completion {
future: Abortable::new(Completion::Future(Box::pin(future)), abort_registration),
abort_handle,
},
}
}
pub fn ready(result: Result<T, Error>) -> Self {
let (sender, receiver) = oneshot::channel();
let _ = sender.send(result);
Self::from_receiver(receiver)
}
pub(crate) fn closed(metric: MetricHandle) -> Self {
metric.finish();
let (sender, receiver) = oneshot::channel();
drop(sender);
Self::from_receiver(receiver)
}
pub fn abort(&self) {
match &self.state {
HandleState::Task {
abort_handle,
metric,
..
} => {
abort_handle.abort();
metric.finish();
}
HandleState::Completion { abort_handle, .. } => {
abort_handle.abort();
}
}
}
pub(crate) fn aborter(&self) -> Option<Aborter> {
match &self.state {
HandleState::Task {
abort_handle,
metric,
..
} => Some(Aborter::new(abort_handle.clone(), metric.clone())),
HandleState::Completion { .. } => None,
}
}
}
impl<T> Future for Handle<T>
where
T: Send + 'static,
{
type Output = Result<T, Error>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match &mut self.state {
HandleState::Task { receiver, .. } => Pin::new(receiver)
.poll(cx)
.map(|result| result.unwrap_or_else(|_| Err(Error::Closed))),
HandleState::Completion { future, .. } => Pin::new(future)
.poll(cx)
.map(|result| result.unwrap_or(Err(Error::Aborted))),
}
}
}
#[derive(Clone)]
pub(crate) struct MetricHandle {
gauge: Gauge,
finished: Arc<Once>,
}
impl MetricHandle {
pub(crate) fn new(gauge: Gauge) -> Self {
gauge.inc();
Self {
gauge,
finished: Arc::new(Once::new()),
}
}
pub(crate) fn finish(&self) {
let gauge = self.gauge.clone();
self.finished.call_once(move || {
gauge.dec();
});
}
}
pub type Panic = Box<dyn Any + Send + 'static>;
#[derive(Clone)]
pub(crate) struct Panicker {
catch: bool,
sender: Arc<Mutex<Option<oneshot::Sender<Panic>>>>,
}
impl Panicker {
pub(crate) fn new(catch: bool) -> (Self, Panicked) {
let (sender, receiver) = oneshot::channel();
let panicker = Self {
catch,
sender: Arc::new(Mutex::new(Some(sender))),
};
let panicked = Panicked { receiver };
(panicker, panicked)
}
#[commonware_macros::stability(ALPHA)]
pub(crate) const fn catch(&self) -> bool {
self.catch
}
pub(crate) fn notify(&self, panic: Box<dyn Any + Send + 'static>) {
let err = extract_panic_message(&*panic);
error!(?err, "task panicked");
if self.catch {
return;
}
let mut sender = self.sender.lock();
let Some(sender) = sender.take() else {
return;
};
let _ = sender.send(panic);
}
}
pub(crate) struct Panicked {
receiver: oneshot::Receiver<Panic>,
}
impl Panicked {
pub(crate) async fn interrupt<Fut>(self, task: Fut) -> Fut::Output
where
Fut: Future,
{
let panicked = self.receiver;
pin_mut!(panicked);
pin_mut!(task);
match select(panicked, task).await {
Either::Left((panic, task)) => match panic {
Ok(panic) => {
resume_unwind(panic);
}
Err(_) => task.await,
},
Either::Right((output, _)) => {
output
}
}
}
}
pub(crate) struct Aborter {
inner: AbortHandle,
metric: MetricHandle,
}
impl Aborter {
pub(crate) const fn new(inner: AbortHandle, metric: MetricHandle) -> Self {
Self { inner, metric }
}
pub(crate) fn abort(self) {
self.inner.abort();
self.metric.finish();
}
}
#[cfg(test)]
mod tests {
use super::Handle;
use crate::{deterministic, Error, Metrics as _, Runner, Spawner, Supervisor as _};
use commonware_utils::channel::oneshot;
use futures::future;
const METRIC_PREFIX: &str = "runtime_tasks_running{";
fn running_tasks_for_label(metrics: &str, label: &str) -> Option<u64> {
let label_fragment = format!("name=\"{label}\"");
metrics.lines().find_map(|line| {
if line.starts_with(METRIC_PREFIX) && line.contains(&label_fragment) {
line.rsplit_once(' ')
.and_then(|(_, value)| value.trim().parse::<u64>().ok())
} else {
None
}
})
}
#[test]
fn tasks_running_decreased_after_completion() {
const LABEL: &str = "tasks_running_after_completion";
let runner = deterministic::Runner::default();
runner.start(|context| async move {
let handle = context.child(LABEL).spawn(|_| async move { "done" });
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(1),
"expected tasks_running gauge to be 1 before completion: {metrics}",
);
let output = handle.await.expect("task failed");
assert_eq!(output, "done");
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(0),
"expected tasks_running gauge to return to 0 after completion: {metrics}",
);
});
}
#[test]
fn tasks_running_unchanged_when_handle_dropped() {
const LABEL: &str = "tasks_running_unchanged";
let runner = deterministic::Runner::default();
runner.start(|context| async move {
let handle = context.child(LABEL).spawn(|_| async move {
future::pending::<()>().await;
});
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(1),
"expected tasks_running gauge to be 1 before dropping handle: {metrics}",
);
drop(handle);
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(1),
"dropping handle should not finish metrics: {metrics}",
);
});
}
#[test]
fn tasks_running_decreased_immediately_on_abort_via_handle() {
const LABEL: &str = "tasks_running_abort_via_handle";
let runner = deterministic::Runner::default();
runner.start(|context| async move {
let handle = context.child(LABEL).spawn(|_| async move {
future::pending::<()>().await;
});
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(1),
"expected tasks_running gauge to be 1 before abort: {metrics}",
);
handle.abort();
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(0),
"expected tasks_running gauge to return to 0 after abort: {metrics}",
);
});
}
#[test]
fn completion_handle_abort_stops_waiting() {
deterministic::Runner::default().start(|_| async move {
let (sender, receiver) = oneshot::channel();
let handle = Handle::from_receiver(receiver);
handle.abort();
assert!(sender.send(Ok(())).is_ok());
assert!(matches!(handle.await, Err(Error::Aborted)));
});
}
#[test]
fn tasks_running_decreased_after_blocking_completion() {
const LABEL: &str = "tasks_running_after_blocking_completion";
let runner = deterministic::Runner::default();
runner.start(|context| async move {
let blocking_handle = context.child(LABEL).shared(true).spawn(|_| async move {
42
});
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(1),
"expected tasks_running gauge to be 1 while blocking task runs: {metrics}",
);
let result = blocking_handle.await.expect("blocking task failed");
assert_eq!(result, 42);
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(0),
"expected tasks_running gauge to return to 0 after blocking task completes: {metrics}",
);
});
}
#[test]
fn tasks_running_decreased_immediately_on_abort_via_aborter() {
const LABEL: &str = "tasks_running_abort_via_aborter";
let runner = deterministic::Runner::default();
runner.start(|context| async move {
let handle = context.child(LABEL).spawn(|_| async move {
future::pending::<()>().await;
});
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(1),
"expected tasks_running gauge to be 1 before abort: {metrics}",
);
let aborter = handle.aborter().unwrap();
aborter.abort();
let metrics = context.encode();
assert_eq!(
running_tasks_for_label(&metrics, LABEL),
Some(0),
"expected tasks_running gauge to return to 0 after abort: {metrics}",
);
});
}
}