use std::cell::RefCell;
use std::future::Future;
use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use super::{InvocationPlugins, OperationInfo, Payload, call_plugin, log_hook_panic};
use crate::context::{ActiveBody, current_body, run_body, run_body_sync};
use crate::{BoxError, WireError};
#[derive(Debug, Clone)]
enum Site {
Invocation,
UserFunction {
info: Arc<OperationInfo>,
started: Arc<[usize]>,
},
}
struct Node {
plugins: Arc<InvocationPlugins>,
parent: Option<Arc<Node>>,
site: Site,
}
impl std::fmt::Debug for Node {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Node")
.field("parent", &self.parent)
.field("site", &self.site)
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, Default)]
pub(crate) struct Frames(Option<Arc<Node>>);
thread_local! {
static CURRENT: RefCell<Frames> = const { RefCell::new(Frames(None)) };
}
struct Restore(Option<Frames>);
impl Drop for Restore {
fn drop(&mut self) {
if let Some(previous) = self.0.take() {
CURRENT.with(|current| *current.borrow_mut() = previous);
}
}
}
fn make_current(frames: &Frames) -> Restore {
Restore(Some(
CURRENT.with(|current| current.replace(frames.clone())),
))
}
impl Frames {
pub(crate) fn none() -> Self {
Self(None)
}
pub(crate) fn current() -> Self {
CURRENT.with(|current| current.borrow().clone())
}
pub(crate) fn invocation(plugins: Option<Arc<InvocationPlugins>>) -> Self {
Self(plugins.map(|plugins| {
Arc::new(Node {
plugins,
parent: None,
site: Site::Invocation,
})
}))
}
#[cfg(test)]
pub(crate) fn with_user_function(&self, info: Arc<OperationInfo>) -> Self {
Self(self.0.as_ref().map(|top| {
Arc::new(Node {
plugins: Arc::clone(&top.plugins),
parent: Some(Arc::clone(top)),
site: Site::UserFunction {
info,
started: (0..top.plugins.plugins.len()).collect(),
},
})
}))
}
pub(crate) fn enter<R>(&self, f: impl FnOnce() -> R) -> R {
self.enter_from(false, f)
}
fn enter_from<R>(&self, only_top: bool, f: impl FnOnce() -> R) -> R {
let Some(top) = &self.0 else {
let _restore = make_current(self);
return f();
};
let outer = current_body();
let mut f = Some(f);
let mut out = None;
let mut innermost = || {
if let Some(f) = f.take() {
let _restore = make_current(self);
out = Some(catch_unwind(AssertUnwindSafe(f)));
}
};
let panics = RefCell::new(Vec::new());
let level = Level {
panics: &panics,
outer: &outer,
};
if only_top {
enter_site(top, &level, &mut innermost);
} else {
enter_node(top, &level, &mut innermost);
}
log_panics(panics);
#[expect(clippy::expect_used)]
let out = out.expect("the innermost call always runs");
match out {
Ok(value) => value,
Err(payload) => resume_unwind(payload),
}
}
pub(crate) fn instrument<F: Future>(&self, fut: F) -> impl Future<Output = F::Output> {
Scoped {
frames: self.clone(),
inner: Some(Box::pin(fut)),
}
}
}
type Panics<'a> = RefCell<Vec<(&'a str, &'static str, Payload)>>;
fn log_panics(panics: Panics<'_>) {
for (identity, hook, payload) in panics.into_inner() {
log_hook_panic(identity, hook, payload);
}
}
struct Level<'a, 'p> {
panics: &'p Panics<'a>,
outer: &'p ActiveBody,
}
fn enter_node<'a>(node: &'a Arc<Node>, level: &Level<'a, '_>, f: &mut dyn FnMut()) {
match &node.parent {
Some(parent) => enter_node(parent, level, &mut || {
enter_site(node, level, f);
}),
None => enter_site(node, level, f),
}
}
fn enter_site<'a>(node: &'a Node, level: &Level<'a, '_>, f: &mut dyn FnMut()) {
match &node.site {
Site::Invocation => {
let mut all = 0..node.plugins.plugins.len();
wrap_each(&mut all, node, Target::Invocation, level, f);
}
Site::UserFunction { info, started } => {
let mut started = started.iter().copied();
wrap_each(&mut started, node, Target::UserFunction(info), level, f);
}
}
}
#[derive(Clone, Copy)]
enum Target<'s> {
Invocation,
UserFunction(&'s OperationInfo),
}
fn wrap_each<'a>(
slots: &mut dyn Iterator<Item = usize>,
node: &'a Node,
target: Target<'_>,
level: &Level<'a, '_>,
f: &mut dyn FnMut(),
) {
let Some(index) = slots.next() else {
f();
return;
};
let mut next = Once::new(|| wrap_each(slots, node, target, level, f));
wrap_one(&node.plugins, index, target, level, &mut next);
}
struct Once<F>(Option<F>);
impl<F: FnOnce()> Once<F> {
fn new(f: F) -> Self {
Self(Some(f))
}
fn call(&mut self) {
if let Some(f) = self.0.take() {
f();
}
}
}
fn wrap_one<'a>(
plugins: &'a InvocationPlugins,
index: usize,
target: Target<'_>,
level: &Level<'a, '_>,
next: &mut Once<impl FnOnce()>,
) {
let slot = plugins.slot(index);
let outer = level.outer;
let mut inside = || run_body_sync(outer.clone(), || next.call());
let (hook, wrapped) = match target {
Target::Invocation => (
"wrap_invocation",
call_plugin(|| slot.plugin.wrap_invocation(plugins.info(), &mut inside)),
),
Target::UserFunction(info) => (
"wrap_user_function",
call_plugin(|| slot.plugin.wrap_user_function(info, &mut inside)),
),
};
if let Err(payload) = wrapped {
level
.panics
.borrow_mut()
.push((slot.identity.as_str(), hook, payload));
}
next.call();
}
struct Scoped<F> {
frames: Frames,
inner: Option<Pin<Box<F>>>,
}
impl<F: Future> Future for Scoped<F> {
type Output = F::Output;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<F::Output> {
let this = self.get_mut();
let slot = &mut this.inner;
this.frames.enter(|| poll_slot(slot, cx))
}
}
fn poll_slot<F: Future>(slot: &mut Option<Pin<Box<F>>>, cx: &mut Context<'_>) -> Poll<F::Output> {
let Some(inner) = slot.as_mut() else {
return Poll::Pending;
};
let Poll::Ready(output) = inner.as_mut().poll(cx) else {
return Poll::Pending;
};
let finished = slot.take();
if let Err(payload) = catch_unwind(AssertUnwindSafe(move || drop(finished))) {
crate::task::drop_after_first_panic(output);
resume_unwind(payload);
}
Poll::Ready(output)
}
impl<F> Drop for Scoped<F> {
fn drop(&mut self) {
if let Some(inner) = self.inner.take() {
self.frames.enter(move || drop(inner));
}
}
}
#[derive(Debug)]
pub(crate) struct Ran<T, E = BoxError>(Outcome<T, E>);
#[derive(Debug)]
enum Outcome<T, E> {
Ok(T),
Err { error: E, wire: WireError },
}
impl<T, E> Ran<T, E> {
#[cfg(test)]
pub(crate) fn from_result(result: Result<T, E>, wire: impl FnOnce(&E) -> WireError) -> Self {
match result {
Ok(value) => Self(Outcome::Ok(value)),
Err(error) => {
let wire = wire(&error);
Self(Outcome::Err { error, wire })
}
}
}
pub(crate) fn outcome(&self) -> Result<(), &WireError> {
match &self.0 {
Outcome::Ok(_) => Ok(()),
Outcome::Err { wire, .. } => Err(wire),
}
}
pub(crate) fn into_parts(self) -> Result<T, (E, WireError)> {
match self.0 {
Outcome::Ok(value) => Ok(value),
Outcome::Err { error, wire } => Err((error, wire)),
}
}
}
#[expect(
clippy::trivially_copy_pass_by_ref,
reason = "a runner's wire function takes its error by reference"
)]
pub(crate) fn never_fails(never: &std::convert::Infallible) -> WireError {
match *never {}
}
pub(crate) fn run_user_function<T, E, B>(
plugins: Option<Arc<InvocationPlugins>>,
info: Arc<OperationInfo>,
active: UserBody,
span: tracing::Span,
body: impl FnOnce() -> B,
wire: impl FnOnce(&E) -> WireError,
) -> impl Future<Output = Ran<T, E>>
where
B: Future<Output = Result<T, E>>,
{
let outcome = active.id().to_owned();
run_user_function_at(plugins, info, active.into(), outcome, span, body, wire)
}
pub(crate) enum UserBody {
Context(String),
Step(String),
}
impl UserBody {
fn id(&self) -> &str {
match self {
Self::Context(id) | Self::Step(id) => id,
}
}
}
impl From<UserBody> for ActiveBody {
fn from(body: UserBody) -> Self {
match body {
UserBody::Context(id) => Self::Context(id),
UserBody::Step(id) => Self::Step(id),
}
}
}
pub(crate) fn run_user_function_at<T, E, B>(
plugins: Option<Arc<InvocationPlugins>>,
info: Arc<OperationInfo>,
active: ActiveBody,
outcome: String,
span: tracing::Span,
body: impl FnOnce() -> B,
wire: impl FnOnce(&E) -> WireError,
) -> impl Future<Output = Ran<T, E>>
where
B: Future<Output = Result<T, E>>,
{
let inner = run_body(active, async move {
let mut body = Some(Box::pin(body()));
match std::future::poll_fn(|cx| poll_slot(&mut body, cx)).await {
Ok(value) => Ran(Outcome::Ok(value)),
Err(error) => {
let wire = run_body_sync(ActiveBody::Outcome(outcome), || wire(&error));
Ran(Outcome::Err { error, wire })
}
}
});
Runner {
info,
plugins,
span,
started: None,
own: None,
inner: Some(Box::pin(inner)),
}
}
struct Runner<F> {
info: Arc<OperationInfo>,
plugins: Option<Arc<InvocationPlugins>>,
span: tracing::Span,
started: Option<Arc<[usize]>>,
own: Option<Arc<Node>>,
inner: Option<Pin<Box<F>>>,
}
fn own_node(
own: &mut Option<Arc<Node>>,
plugins: &Arc<InvocationPlugins>,
info: &Arc<OperationInfo>,
started: Arc<[usize]>,
) -> Frames {
let parent = Frames::current().0;
if let Some(node) = own.as_ref()
&& same_node(node.parent.as_ref(), parent.as_ref())
&& matches!(&node.site, Site::UserFunction { started: kept, .. } if Arc::ptr_eq(kept, &started))
{
return Frames(Some(Arc::clone(node)));
}
let node = Arc::new(Node {
plugins: Arc::clone(plugins),
parent,
site: Site::UserFunction {
info: Arc::clone(info),
started,
},
});
*own = Some(Arc::clone(&node));
Frames(Some(node))
}
fn same_node(a: Option<&Arc<Node>>, b: Option<&Arc<Node>>) -> bool {
match (a, b) {
(Some(a), Some(b)) => Arc::ptr_eq(a, b),
(None, None) => true,
_ => false,
}
}
impl<T, E, F: Future<Output = Ran<T, E>>> Future for Runner<F> {
type Output = Ran<T, E>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Ran<T, E>> {
let this = self.get_mut();
let span = this.span.clone();
match crate::task::catch_in_span(&span, || this.poll_in_span(cx)) {
Ok(polled) => polled,
Err(payload) => resume_unwind(payload),
}
}
}
impl<T, E, F: Future<Output = Ran<T, E>>> Runner<F> {
fn poll_in_span(&mut self, cx: &mut Context<'_>) -> Polled<Ran<T, E>> {
let this = self;
let first = this.started.is_none();
let Some(plugins) = this.plugins.clone() else {
this.started = Some(Arc::from([]));
return catch_unwind(AssertUnwindSafe(|| poll_slot(&mut this.inner, cx)));
};
let candidates: Arc<[usize]> = match &this.started {
Some(started) => Arc::clone(started),
None => (0..plugins.plugins.len()).collect(),
};
let outer = current_body();
let panics = RefCell::new(Vec::new());
let chain = OwnChain {
plugins: &plugins,
info: &this.info,
candidates: &candidates,
first,
started: RefCell::new(Vec::new()),
published: RefCell::new(None),
out: RefCell::new(None),
level: Level {
panics: &panics,
outer: &outer,
},
};
let (inner, own, info) = (&mut this.inner, &mut this.own, &this.info);
let mut innermost = |started: Arc<[usize]>| {
let node = own_node(own, &plugins, info, started);
let _restore = make_current(&node);
catch_unwind(AssertUnwindSafe(|| poll_slot(inner, cx)))
};
chain.level(0, &mut innermost);
let OwnChain { published, out, .. } = chain;
#[expect(clippy::expect_used)]
let published = published
.into_inner()
.expect("the innermost call always runs");
this.started = Some(published);
log_panics(panics);
#[expect(clippy::expect_used)]
out.into_inner().expect("the innermost call always runs")
}
}
impl<F> Drop for Runner<F> {
fn drop(&mut self) {
let dropped = match self.inner.take() {
Some(inner) => {
let Self {
span,
started,
plugins,
own,
info,
..
} = self;
crate::task::catch_in_span(span, || {
catch_unwind(AssertUnwindSafe(|| {
match (started.clone(), plugins.as_ref()) {
(Some(started), Some(plugins)) => {
own_node(own, plugins, info, started)
.enter_from(true, move || drop(inner));
}
_ => drop(inner),
}
}))
})
}
None => Ok(()),
};
let span = std::mem::replace(&mut self.span, tracing::Span::none());
let closed = catch_unwind(AssertUnwindSafe(move || drop(span)));
match (dropped, closed) {
(Err(first), closed) => {
if let Err(later) = closed {
crate::task::discard_panic(later);
}
resume_unwind(first);
}
(Ok(()), Err(close)) => resume_unwind(close),
(Ok(()), Ok(())) => {}
}
}
}
type Polled<T> = std::thread::Result<Poll<T>>;
struct OwnChain<'a, 'p, T, E> {
plugins: &'a InvocationPlugins,
info: &'p Arc<OperationInfo>,
candidates: &'p Arc<[usize]>,
first: bool,
started: RefCell<Vec<usize>>,
published: RefCell<Option<Arc<[usize]>>>,
out: RefCell<Option<Polled<Ran<T, E>>>>,
level: Level<'a, 'p>,
}
impl<T, E> OwnChain<'_, '_, T, E> {
fn level(&self, k: usize, innermost: &mut dyn FnMut(Arc<[usize]>) -> Polled<Ran<T, E>>) {
let Some(&index) = self.candidates.get(k) else {
let started: Arc<[usize]> = if self.first {
self.started.borrow().as_slice().into()
} else {
Arc::clone(self.candidates)
};
*self.published.borrow_mut() = Some(Arc::clone(&started));
let polled = innermost(started);
*self.out.borrow_mut() = Some(polled);
return;
};
if self.first {
let ok = self
.plugins
.dispatch_to("on_user_function_start", [index], |plugin| {
plugin.on_user_function_start(self.info)
});
if ok.is_empty() {
self.level(k + 1, innermost);
return;
}
self.started.borrow_mut().push(index);
}
let mut next = Once::new(|| self.level(k + 1, innermost));
let target = Target::UserFunction(self.info);
wrap_one(self.plugins, index, target, &self.level, &mut next);
if let Some(Ok(Poll::Ready(ran))) = &*self.out.borrow() {
let outcome = ran.outcome();
self.plugins
.dispatch_to("on_user_function_end", [index], |plugin| {
plugin.on_user_function_end(self.info, outcome)
});
}
}
}
#[cfg(test)]
mod tests {
use std::cell::Cell;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use super::*;
use crate::plugin::test_support::{capture, field, warnings};
use crate::plugin::{
HookWork, InvocationInfo, Plugin, PluginFactory, WorkState, create_plugins,
};
thread_local! {
static ACTIVE: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
}
fn active() -> Vec<String> {
ACTIVE.with(|a| a.borrow().clone())
}
fn label(info: &OperationInfo) -> String {
info.name().unwrap_or(info.id()).to_owned()
}
fn frame_ids() -> Vec<String> {
let mut ids = Vec::new();
let mut node = Frames::current().0;
while let Some(current) = node {
ids.push(match ¤t.site {
Site::Invocation => "I".to_owned(),
Site::UserFunction { info, .. } => label(info),
});
node = current.parent.clone();
}
ids.reverse();
ids
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum Mode {
Normal,
Skip,
Twice,
PanicBefore,
PanicAfter,
StartFails,
StartPanics,
StartPending,
StartFutureFails,
StartFuturePanics,
EndFails,
EndPanics,
EndFutureFails,
EndFuturePanics,
}
struct PanicsWhenPolled(&'static str);
impl Future for PanicsWhenPolled {
type Output = Result<(), BoxError>;
#[expect(clippy::panic, reason = "the test needs a future that panics")]
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
std::panic::panic_any(self.0)
}
}
fn polls(pending: usize, mut on_poll: impl FnMut()) -> impl Future<Output = ()> {
let mut left = pending;
std::future::poll_fn(move |cx| {
on_poll();
if left == 0 {
Poll::Ready(())
} else {
left -= 1;
cx.waker().wake_by_ref();
Poll::Pending
}
})
}
struct Recorder {
tag: &'static str,
mode: Mode,
log: Arc<Mutex<Vec<String>>>,
}
impl Recorder {
fn wrap(&self, site: &str, next: &mut dyn FnMut()) {
let tag = format!("{}:{site}", self.tag);
self.log.lock().unwrap().push(format!("enter {tag}"));
assert!(self.mode != Mode::PanicBefore, "B");
if self.mode != Mode::Skip {
ACTIVE.with(|a| a.borrow_mut().push(tag.clone()));
next();
if self.mode == Mode::Twice {
next();
}
ACTIVE.with(|a| a.borrow_mut().pop());
}
self.log.lock().unwrap().push(format!("exit {tag}"));
assert!(self.mode != Mode::PanicAfter, "B");
}
}
impl Plugin for Recorder {
fn name(&self) -> &'static str {
self.tag
}
fn wrap_invocation(&self, _info: &InvocationInfo, next: &mut dyn FnMut()) {
self.wrap("inv", next);
}
fn wrap_user_function(&self, info: &OperationInfo, next: &mut dyn FnMut()) {
self.wrap(&label(info), next);
}
#[expect(clippy::panic, reason = "one mode tests a panicking start hook")]
fn on_user_function_start(&self, info: &OperationInfo) -> HookWork {
let line = format!("start {}:{} in {:?}", self.tag, label(info), active());
self.log.lock().unwrap().push(line);
match self.mode {
Mode::StartFails => Err("start failed".into()),
Mode::StartPanics => std::panic::panic_any("start panicked"),
Mode::StartPending => Ok(Some(Box::pin(std::future::pending()))),
Mode::StartFutureFails => Ok(Some(Box::pin(async { Err("future failed".into()) }))),
Mode::StartFuturePanics => Ok(Some(Box::pin(PanicsWhenPolled("future panicked")))),
_ => Ok(None),
}
}
#[expect(clippy::panic, reason = "one mode tests a panicking end hook")]
fn on_user_function_end(
&self,
info: &OperationInfo,
outcome: Result<(), &WireError>,
) -> HookWork {
let result = match outcome {
Ok(()) => "ok".to_owned(),
Err(e) => format!("err {}", e.error_message().unwrap_or_default()),
};
let line = format!(
"end {}:{} {result} in {:?}",
self.tag,
label(info),
active()
);
self.log.lock().unwrap().push(line);
match self.mode {
Mode::EndFails => Err("end failed".into()),
Mode::EndPanics => std::panic::panic_any("end panicked"),
Mode::EndFutureFails => Ok(Some(Box::pin(async { Err("future failed".into()) }))),
Mode::EndFuturePanics => Ok(Some(Box::pin(PanicsWhenPolled("future panicked")))),
_ => Ok(None),
}
}
}
struct Factory {
tag: &'static str,
mode: Mode,
log: Arc<Mutex<Vec<String>>>,
}
impl PluginFactory for Factory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(Recorder {
tag: self.tag,
mode: self.mode,
log: Arc::clone(&self.log),
})))
}
}
fn plugins(
modes: &[(&'static str, Mode)],
log: &Arc<Mutex<Vec<String>>>,
) -> Arc<InvocationPlugins> {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> = modes
.iter()
.map(|&(tag, mode)| {
let factory: Arc<dyn PluginFactory> = Arc::new(Factory {
tag,
mode,
log: Arc::clone(log),
});
("Factory", factory)
})
.collect();
create_plugins(&factories, None, crate::plugin::tests_support_info)
.expect("factories were given")
}
fn info(id: &str) -> Arc<OperationInfo> {
Arc::new(OperationInfo::for_test(id))
}
fn frames(plugins: &Arc<InvocationPlugins>, ids: &[&str]) -> Frames {
let mut frames = Frames::invocation(Some(Arc::clone(plugins)));
for id in ids {
frames = frames.with_user_function(info(id));
}
frames
}
fn inv(plugins: &Arc<InvocationPlugins>) -> Frames {
Frames::invocation(Some(Arc::clone(plugins)))
}
fn take(log: &Arc<Mutex<Vec<String>>>) -> Vec<String> {
std::mem::take(&mut *log.lock().unwrap())
}
fn push(log: &Arc<Mutex<Vec<String>>>, line: String) {
log.lock().unwrap().push(line);
}
fn wrap_lines(log: &Arc<Mutex<Vec<String>>>) -> Vec<String> {
take(log)
.into_iter()
.filter(|l| !l.starts_with("start") && !l.starts_with("end"))
.collect()
}
fn step_runner<T, B>(
plugins: &Arc<InvocationPlugins>,
body: impl FnOnce() -> B,
) -> impl Future<Output = Ran<T>>
where
B: Future<Output = Result<T, BoxError>>,
{
run_user_function(
Some(Arc::clone(plugins)),
info("s"),
UserBody::Step("s".to_owned()),
tracing::Span::none(),
body,
|e: &BoxError| WireError::new(Some("Error".to_owned()), Some(e.to_string())),
)
}
async fn settle_work(plugins: &InvocationPlugins) {
loop {
let handles = match &mut *plugins.lock_work() {
WorkState::Open(list) => std::mem::take(list),
WorkState::Closed => Vec::new(),
};
if handles.is_empty() {
return;
}
for handle in handles {
let _ = handle.await;
}
}
}
#[test]
fn wraps_nest_by_frame_then_registration() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal), ("p1", Mode::Normal)], &log);
let body_log = Arc::clone(&log);
let mut run = std::pin::pin!(step_runner(&plugins, move || async move {
polls(1, || push(&body_log, format!("body in {:?}", frame_ids()))).await;
Ok::<u32, BoxError>(1)
}));
let poll = [
"enter p0:inv",
"enter p1:inv",
"enter p0:c",
"enter p1:c",
"enter p0:s",
"enter p1:s",
"body in [\"I\", \"c\", \"s\"]",
"exit p1:s",
"exit p0:s",
"exit p1:c",
"exit p0:c",
"exit p1:inv",
"exit p0:inv",
];
let c = frames(&plugins, &["c"]);
assert!(c.enter(|| futures_poll_once(run.as_mut())).is_pending());
assert_eq!(wrap_lines(&log), poll);
assert!(c.enter(|| futures_poll_once(run.as_mut())).is_ready());
assert_eq!(wrap_lines(&log), poll);
}
#[test]
fn hooks_interleave_with_wraps() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal), ("p1", Mode::Normal)], &log);
let body_log = Arc::clone(&log);
let mut run = std::pin::pin!(step_runner(&plugins, move || {
push(&body_log, "closure called".to_owned());
async move {
push(&body_log, "body".to_owned());
Ok::<u32, BoxError>(1)
}
}));
let Poll::Ready(ran) = futures_poll_once(run.as_mut()) else {
unreachable!("the body is done in its first poll");
};
assert_eq!(ran.into_parts().ok(), Some(1));
assert_eq!(
take(&log),
[
"start p0:s in []",
"enter p0:s",
"start p1:s in [\"p0:s\"]",
"enter p1:s",
"closure called",
"body",
"exit p1:s",
"end p1:s ok in [\"p0:s\"]",
"exit p0:s",
"end p0:s ok in []",
]
);
}
#[test]
fn runner_polled_under_no_frames() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal)], &log);
let body = |log: &Arc<Mutex<Vec<String>>>| {
let log = Arc::clone(log);
move || async move {
polls(1, || {
push(&log, format!("body {:?} in {:?}", frame_ids(), active()));
})
.await;
Ok::<u32, BoxError>(3)
}
};
let read = "body [\"s\"] in [\"p0:s\"]";
let mut run = std::pin::pin!(step_runner(&plugins, body(&log)));
assert!(Frames::current().0.is_none());
assert!(futures_poll_once(run.as_mut()).is_pending());
assert!(futures_poll_once(run.as_mut()).is_ready());
assert_eq!(
take(&log),
[
"start p0:s in []",
"enter p0:s",
read,
"exit p0:s",
"enter p0:s",
read,
"exit p0:s",
"end p0:s ok in []",
]
);
let mut run = std::pin::pin!(step_runner(&plugins, body(&log)));
let a = frames(&plugins, &["a"]);
assert!(a.enter(|| futures_poll_once(run.as_mut())).is_pending());
drop(take(&log));
assert!(futures_poll_once(run.as_mut()).is_ready());
assert_eq!(
take(&log),
["enter p0:s", read, "exit p0:s", "end p0:s ok in []"]
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn map_item_factory_runs_inside_span() {
let log = Arc::new(Mutex::new(Vec::new()));
let (buffer, _guard) = capture();
let options = crate::Options::builder()
.plugin(Factory {
tag: "p0",
mode: Mode::Normal,
log: Arc::clone(&log),
})
.plugin(Factory {
tag: "p1",
mode: Mode::Normal,
log: Arc::clone(&log),
})
.build()
.expect("valid options");
let result = crate::test_util::LocalRunner::new()
.options(options)
.run(
|_event: (), ctx: crate::DurableContext| async move {
let values = ctx
.map(vec![1_u32], |_child, item, _idx| {
tracing::info!(target: "probe", "factory");
async move { Ok(item) }
})
.name("m")
.await?;
Ok::<_, BoxError>(values)
},
(),
)
.await
.expect("runner");
assert_eq!(result.output(), Some(&vec![1]));
let item_id = result
.operation("map-item-0")
.expect("the item is recorded")
.id()
.to_owned();
let events: Vec<serde_json::Value> = String::from_utf8_lossy(&buffer.lock().unwrap())
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.filter(|v| field(v, "target") == "probe")
.collect();
assert_eq!(events.len(), 1, "{events:?}");
let span = events
.first()
.and_then(|e| e.get("span"))
.expect("the event has a span");
assert_eq!(field(span, "name"), "durable_execution", "{span}");
assert_eq!(field(span, "operationName"), "map-item-0", "{span}");
assert_eq!(field(span, "operationId"), item_id, "{span}");
}
#[derive(Debug)]
struct Dc {
calls: Arc<AtomicUsize>,
seen: Arc<Mutex<Vec<String>>>,
}
impl std::fmt::Display for Dc {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.calls.fetch_add(1, Ordering::Relaxed);
push(
&self.seen,
format!("display in {:?} under {:?}", span_name(), current_body()),
);
f.write_str("dc")
}
}
impl std::error::Error for Dc {}
fn span_name() -> Option<&'static str> {
tracing::Span::current()
.metadata()
.map(tracing::Metadata::name)
}
#[test]
fn wire_derived_inside_span() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal)], &log);
let seen = Arc::new(Mutex::new(Vec::new()));
let calls = Arc::new(AtomicUsize::new(0));
let (factory, polled) = (Arc::clone(&seen), Arc::clone(&seen));
let error = Dc {
calls: Arc::clone(&calls),
seen: Arc::clone(&seen),
};
let ran = {
let _guard = test_tracing::install_test_subscriber(tracing_subscriber::registry());
let outer = tracing::info_span!("outer");
let _entered = outer.enter();
let mut run = std::pin::pin!(run_user_function(
Some(Arc::clone(&plugins)),
info("m"),
UserBody::Context("m".to_owned()),
tracing::info_span!("body_span"),
move || {
push(&factory, format!("factory in {:?}", span_name()));
async move {
push(&polled, format!("poll in {:?}", span_name()));
Err::<(), BoxError>(Box::new(error))
}
},
|e: &BoxError| WireError::new(Some("Error".to_owned()), Some(e.to_string())),
));
let Poll::Ready(ran) = futures_poll_once(run.as_mut()) else {
unreachable!("the body fails in its first poll");
};
ran
};
assert_eq!(calls.load(Ordering::Relaxed), 1);
assert_eq!(
take(&seen),
[
"factory in Some(\"body_span\")",
"poll in Some(\"body_span\")",
"display in Some(\"body_span\") under Outcome(\"m\")",
]
);
let (_, wire) = ran.into_parts().expect_err("the body failed");
assert_eq!(wire.error_message(), Some("dc"));
assert!(
take(&log).contains(&"end p0:m err dc in []".to_owned()),
"the end hook received the wire"
);
}
#[test]
fn runner_polled_under_two_parent_chains() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal)], &log);
let body_log = Arc::clone(&log);
let mut run = std::pin::pin!(step_runner(&plugins, move || async move {
polls(1, || push(&body_log, format!("body in {:?}", frame_ids()))).await;
Ok::<u32, BoxError>(3)
}));
let first = frames(&plugins, &["a"]).enter(|| futures_poll_once(run.as_mut()));
assert!(first.is_pending());
let Poll::Ready(ran) = frames(&plugins, &["b"]).enter(|| futures_poll_once(run.as_mut()))
else {
unreachable!("the body returns at its second poll");
};
assert_eq!(ran.into_parts().ok(), Some(3));
assert_eq!(
wrap_lines(&log),
[
"enter p0:inv",
"enter p0:a",
"enter p0:s",
"body in [\"I\", \"a\", \"s\"]",
"exit p0:s",
"exit p0:a",
"exit p0:inv",
"enter p0:inv",
"enter p0:b",
"enter p0:s",
"body in [\"I\", \"b\", \"s\"]",
"exit p0:s",
"exit p0:b",
"exit p0:inv",
]
);
}
type Call = Arc<dyn Fn() + Send + Sync>;
type Work = Arc<dyn Fn() -> HookWork + Send + Sync>;
#[derive(Clone, Default)]
struct Calls {
name: Option<Call>,
wrap_invocation: Option<Call>,
wrap_user_function: Option<Call>,
user_function_start: Option<Work>,
user_function_end: Option<Work>,
operation_start: Option<Work>,
invocation_start: Option<Call>,
}
impl Plugin for Calls {
fn name(&self) -> &'static str {
if let Some(name) = &self.name {
name();
}
"calls"
}
fn wrap_invocation(&self, _info: &InvocationInfo, next: &mut dyn FnMut()) {
if let Some(wrap) = &self.wrap_invocation {
wrap();
}
next();
}
fn wrap_user_function(&self, _info: &OperationInfo, next: &mut dyn FnMut()) {
if let Some(wrap) = &self.wrap_user_function {
wrap();
}
next();
}
fn on_user_function_start(&self, _info: &OperationInfo) -> HookWork {
self.user_function_start.as_ref().map_or(Ok(None), |f| f())
}
fn on_user_function_end(
&self,
_info: &OperationInfo,
_outcome: Result<(), &WireError>,
) -> HookWork {
self.user_function_end.as_ref().map_or(Ok(None), |f| f())
}
fn on_operation_start(&self, _info: &OperationInfo) -> HookWork {
self.operation_start.as_ref().map_or(Ok(None), |f| f())
}
fn on_invocation_start<'a>(
&'a self,
_info: &'a InvocationInfo,
) -> crate::plugin::HookFuture<'a> {
if let Some(start) = &self.invocation_start {
start();
}
Box::pin(std::future::ready(Ok(())))
}
}
impl PluginFactory for Calls {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
fn calls_plugins(calls: Calls) -> Arc<InvocationPlugins> {
let factory: Arc<dyn PluginFactory> = Arc::new(calls);
create_plugins(
&[("Calls", factory)],
None,
crate::plugin::tests_support_info,
)
.expect("a factory was given")
}
#[test]
fn start_hook_inside_sdk_span() {
let seen = Arc::new(Mutex::new(Vec::new()));
let start_seen = Arc::clone(&seen);
let plugins = calls_plugins(Calls {
user_function_start: Some(Arc::new(move || {
push(&start_seen, format!("start in {:?}", span_name()));
Ok(None)
})),
..Calls::default()
});
let body_seen = Arc::clone(&seen);
{
let _guard = test_tracing::install_test_subscriber(tracing_subscriber::registry());
let outer = tracing::info_span!("outer");
let _entered = outer.enter();
let mut run = std::pin::pin!(run_user_function(
Some(Arc::clone(&plugins)),
info("s"),
UserBody::Step("s".to_owned()),
tracing::info_span!("durable_operation"),
move || async move {
tracing::info!("body event");
push(&body_seen, format!("body in {:?}", span_name()));
Ok::<(), BoxError>(())
},
|_: &BoxError| WireError::new(None::<String>, None::<String>),
));
assert!(futures_poll_once(run.as_mut()).is_ready());
};
assert_eq!(
take(&seen),
[
"start in Some(\"durable_operation\")",
"body in Some(\"durable_operation\")",
]
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn wrap_without_next_falls_through() {
let log = Arc::new(Mutex::new(Vec::new()));
let seen = Arc::new(Mutex::new(Vec::new()));
let (buffer, _guard) = capture();
let options = crate::Options::builder()
.plugin(Factory {
tag: "p0",
mode: Mode::Skip,
log: Arc::clone(&log),
})
.plugin(Factory {
tag: "p1",
mode: Mode::Normal,
log: Arc::clone(&log),
})
.build()
.expect("valid options");
let handler_seen = Arc::clone(&seen);
let result = crate::test_util::LocalRunner::new()
.options(options)
.run(
move |_event: (), ctx: crate::DurableContext| {
let seen = Arc::clone(&handler_seen);
async move {
push(&seen, format!("handler in {:?}", active()));
let body_seen = Arc::clone(&seen);
ctx.step(move |_| async move {
push(&body_seen, format!("body in {:?}", active()));
Ok(())
})
.name("s")
.await?;
Ok::<_, BoxError>("done".to_owned())
}
},
(),
)
.await
.expect("runner");
assert_eq!(result.output().map(String::as_str), Some("done"));
assert_eq!(
take(&seen),
["handler in [\"p1:inv\"]", "body in [\"p1:inv\", \"p1:s\"]",]
);
assert!(warnings(&buffer).is_empty(), "{:?}", warnings(&buffer));
}
#[test]
fn second_next_call_does_nothing() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Twice), ("p1", Mode::Normal)], &log);
let runs = Cell::new(0);
frames(&plugins, &[]).enter(|| runs.set(runs.get() + 1));
assert_eq!(runs.get(), 1);
assert_eq!(
take(&log),
["enter p0:inv", "enter p1:inv", "exit p1:inv", "exit p0:inv"]
);
let body_polls = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&body_polls);
let mut run = std::pin::pin!(step_runner(&plugins, move || async move {
polls(1, || {
counter.fetch_add(1, Ordering::Relaxed);
})
.await;
Ok::<u32, BoxError>(1)
}));
assert!(futures_poll_once(run.as_mut()).is_pending());
assert_eq!(body_polls.load(Ordering::Relaxed), 1);
assert!(futures_poll_once(run.as_mut()).is_ready());
assert_eq!(body_polls.load(Ordering::Relaxed), 2);
}
fn assert_one_panic_warning(buffer: &Arc<Mutex<Vec<u8>>>, hook: &str) {
let warnings = warnings(buffer);
assert_eq!(warnings.len(), 1, "{warnings:?}");
let warning = warnings.first().expect("one warning");
assert_eq!(
field(warning, "message"),
"plugin hook panicked; the panic is ignored"
);
assert_eq!(field(warning, "hook"), hook);
assert!(
field(warning, "plugin").ends_with("(plugins[0])"),
"{warning}"
);
}
#[test]
fn wrap_panics_are_contained() {
for mode in [Mode::PanicBefore, Mode::PanicAfter] {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", mode), ("p1", Mode::Normal)], &log);
let (buffer, guard) = capture();
let runs = Cell::new(0);
let value = inv(&plugins).enter(|| {
runs.set(runs.get() + 1);
push(&log, format!("handler in {:?}", active()));
"handler"
});
drop(guard);
assert_eq!((value, runs.get()), ("handler", 1));
let expected: &[&str] = if mode == Mode::PanicBefore {
&[
"enter p0:inv",
"enter p1:inv",
"handler in [\"p1:inv\"]",
"exit p1:inv",
]
} else {
&[
"enter p0:inv",
"enter p1:inv",
"handler in [\"p0:inv\", \"p1:inv\"]",
"exit p1:inv",
"exit p0:inv",
]
};
assert_eq!(take(&log), expected, "{mode:?}");
assert_one_panic_warning(&buffer, "wrap_invocation");
let (buffer, guard) = capture();
let body_log = Arc::clone(&log);
let mut run = std::pin::pin!(step_runner(&plugins, move || async move {
push(&body_log, format!("body in {:?}", active()));
Ok::<u32, BoxError>(7)
}));
let Poll::Ready(ran) = futures_poll_once(run.as_mut()) else {
unreachable!("the body is done in its first poll");
};
drop(guard);
assert_eq!(ran.into_parts().ok(), Some(7));
let expected: &[&str] = if mode == Mode::PanicBefore {
&[
"start p0:s in []",
"enter p0:s",
"start p1:s in []",
"enter p1:s",
"body in [\"p1:s\"]",
"exit p1:s",
"end p1:s ok in []",
"end p0:s ok in []",
]
} else {
&[
"start p0:s in []",
"enter p0:s",
"start p1:s in [\"p0:s\"]",
"enter p1:s",
"body in [\"p0:s\", \"p1:s\"]",
"exit p1:s",
"end p1:s ok in [\"p0:s\"]",
"exit p0:s",
"end p0:s ok in []",
]
};
assert_eq!(take(&log), expected, "{mode:?}");
assert_one_panic_warning(&buffer, "wrap_user_function");
}
}
#[test]
#[expect(clippy::panic, reason = "the test body panics on purpose")]
fn body_panic_resumes_after_every_wrap_returned() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal), ("p1", Mode::Normal)], &log);
let c = frames(&plugins, &["c"]);
let caught = catch_unwind(AssertUnwindSafe(|| {
c.enter(|| -> u32 { std::panic::panic_any("kaboom") })
}));
let payload = caught.expect_err("the body panicked");
assert_eq!(payload.downcast_ref::<&str>(), Some(&"kaboom"));
assert_eq!(
take(&log),
[
"enter p0:inv",
"enter p1:inv",
"enter p0:c",
"enter p1:c",
"exit p1:c",
"exit p0:c",
"exit p1:inv",
"exit p0:inv",
]
);
assert_eq!(active(), Vec::<String>::new());
let mut run = std::pin::pin!(step_runner(&plugins, || PanicsWhenPolled("kaboom")));
let caught = catch_unwind(AssertUnwindSafe(|| {
c.enter(|| futures_poll_once(run.as_mut()))
}));
let payload = caught.expect_err("the body panicked");
assert_eq!(payload.downcast_ref::<&str>(), Some(&"kaboom"));
let lines = take(&log);
assert!(!lines.iter().any(|l| l.starts_with("end")), "{lines:?}");
assert_eq!(lines.last().map(String::as_str), Some("exit p0:inv"));
assert!(lines.contains(&"exit p0:s".to_owned()), "{lines:?}");
}
fn assert_one_hook_warning(buffer: &Arc<Mutex<Vec<u8>>>, hook: &str, error: Option<&str>) {
let warnings = warnings(buffer);
assert_eq!(warnings.len(), 1, "{hook}: {warnings:?}");
let warning = warnings.first().expect("one warning");
assert_eq!(field(warning, "plugin"), "p0 (plugins[0])", "{warning}");
assert_eq!(field(warning, "hook"), hook, "{warning}");
match error {
Some(error) => {
assert_eq!(
field(warning, "message"),
"plugin hook failed; the error is ignored"
);
assert_eq!(field(warning, "error"), error, "{warning}");
}
None => assert_eq!(
field(warning, "message"),
"plugin hook panicked; the panic is ignored"
),
}
}
#[tokio::test]
async fn failing_hook_future_keeps_the_plugin_started() {
for (mode, hook, error) in [
(
Mode::StartFutureFails,
"on_user_function_start",
Some("future failed"),
),
(Mode::StartFuturePanics, "on_user_function_start", None),
(
Mode::EndFutureFails,
"on_user_function_end",
Some("future failed"),
),
(Mode::EndFuturePanics, "on_user_function_end", None),
] {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", mode), ("p1", Mode::Normal)], &log);
let (buffer, _guard) = capture();
let ran: Ran<u32> = step_runner(&plugins, || async { Ok(7) }).await;
settle_work(&plugins).await;
assert_eq!(ran.into_parts().ok(), Some(7));
let lines = take(&log);
for line in ["enter p0:s", "exit p0:s", "end p0:s ok in []"] {
assert!(lines.contains(&line.to_owned()), "{mode:?}: {lines:?}");
}
assert_one_hook_warning(&buffer, hook, error);
}
}
#[tokio::test]
async fn failed_user_function_hook_is_contained() {
for (mode, hook, error) in [
(
Mode::StartFails,
"on_user_function_start",
Some("start failed"),
),
(Mode::StartPanics, "on_user_function_start", None),
(Mode::EndFails, "on_user_function_end", Some("end failed")),
(Mode::EndPanics, "on_user_function_end", None),
] {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", mode), ("p1", Mode::Normal)], &log);
let (buffer, _guard) = capture();
let runs = AtomicUsize::new(0);
let body_log = Arc::clone(&log);
let ran: Ran<u32> = step_runner(&plugins, || {
runs.fetch_add(1, Ordering::Relaxed);
async move {
push(&body_log, format!("body in {:?}", active()));
Ok(7)
}
})
.await;
assert_eq!(ran.into_parts().ok(), Some(7), "{mode:?}");
assert_eq!(runs.load(Ordering::Relaxed), 1);
let lines = take(&log);
if hook == "on_user_function_start" {
assert_eq!(
lines,
[
"start p0:s in []",
"start p1:s in []",
"enter p1:s",
"body in [\"p1:s\"]",
"exit p1:s",
"end p1:s ok in []",
],
"{mode:?}"
);
} else {
assert!(
lines.contains(&"end p0:s ok in []".to_owned()),
"{mode:?}: {lines:?}"
);
}
assert_one_hook_warning(&buffer, hook, error);
}
}
#[test]
fn end_hook_runs_in_the_poll_of_the_return() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal), ("p1", Mode::Normal)], &log);
let body_log = Arc::clone(&log);
let mut run = std::pin::pin!(step_runner(&plugins, move || async move {
push(&body_log, "body".to_owned());
polls(1, || {}).await;
Ok::<u32, BoxError>(1)
}));
assert!(futures_poll_once(run.as_mut()).is_pending());
assert_eq!(
take(&log),
[
"start p0:s in []",
"enter p0:s",
"start p1:s in [\"p0:s\"]",
"enter p1:s",
"body",
"exit p1:s",
"exit p0:s",
]
);
let Poll::Ready(ran) = futures_poll_once(run.as_mut()) else {
unreachable!("the body returns at its second poll");
};
assert_eq!(ran.into_parts().ok(), Some(1));
assert_eq!(
take(&log),
[
"enter p0:s",
"enter p1:s",
"exit p1:s",
"end p1:s ok in [\"p0:s\"]",
"exit p0:s",
"end p0:s ok in []",
]
);
}
#[tokio::test]
async fn pending_start_future_does_not_delay_the_body() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::StartPending), ("p1", Mode::Normal)], &log);
let mut run = std::pin::pin!(step_runner(&plugins, || async { Ok::<u32, BoxError>(1) }));
let Poll::Ready(ran) = inv(&plugins).enter(|| futures_poll_once(run.as_mut())) else {
unreachable!("the runner does not wait for the start future");
};
assert_eq!(ran.into_parts().ok(), Some(1));
for _ in 0..8 {
tokio::task::yield_now().await;
}
let WorkState::Open(list) = &*plugins.lock_work() else {
unreachable!("the drain has not closed");
};
assert_eq!(list.len(), 1, "the start future runs on its own task");
assert!(!list.iter().any(tokio::task::JoinHandle::is_finished));
for handle in list {
handle.abort();
}
}
struct CallsOnDisplay(Call);
impl std::fmt::Debug for CallsOnDisplay {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("CallsOnDisplay")
}
}
impl std::fmt::Display for CallsOnDisplay {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
(self.0)();
f.write_str("calls on display")
}
}
impl std::error::Error for CallsOnDisplay {}
struct CallsOnDrop(Call);
impl std::fmt::Debug for CallsOnDrop {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("CallsOnDrop")
}
}
impl std::fmt::Display for CallsOnDrop {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("calls on drop")
}
}
impl std::error::Error for CallsOnDrop {}
impl Drop for CallsOnDrop {
fn drop(&mut self) {
(self.0)();
}
}
struct CallingFactory(Call);
impl PluginFactory for CallingFactory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
(self.0)();
Ok(None)
}
}
async fn exercise(plugins: &Arc<InvocationPlugins>) {
plugins.invocation_start().await;
let ran: Ran<()> = inv(plugins)
.instrument(step_runner(plugins, || async { Ok(()) }))
.await;
assert!(ran.into_parts().is_ok());
let op = OperationInfo::for_test("s");
plugins.dispatch_work("on_operation_start", |p| p.on_operation_start(&op));
settle_work(plugins).await;
}
fn assert_context_usage_error(ctx: &crate::DurableContext, case: &str) {
let fatal = ctx.coordinator().fatal_error();
let message = fatal.map(|f| f.error_message).unwrap_or_default();
assert_eq!(
message,
"context usage error in \"wait\": the operation was called inside plugin \
code. Plugin code cannot call durable operations.",
"{case}"
);
}
#[tokio::test]
#[expect(clippy::panic, reason = "one case panics with a payload")]
#[expect(clippy::too_many_lines, reason = "one case per hook family")]
async fn plugin_guard_covers_every_hook_family() {
type Case = (&'static str, fn(Call) -> Calls);
fn work(call: Call) -> Work {
Arc::new(move || {
call();
Ok(None)
})
}
fn future(call: Call) -> Work {
Arc::new(move || {
let call = Arc::clone(&call);
Ok(Some(Box::pin(async move {
call();
Ok(())
})))
})
}
let cases: [Case; 13] = [
("wrap_invocation", |c| Calls {
wrap_invocation: Some(c),
..Calls::default()
}),
("wrap_user_function", |c| Calls {
wrap_user_function: Some(c),
..Calls::default()
}),
("on_user_function_start", |c| Calls {
user_function_start: Some(work(c)),
..Calls::default()
}),
("on_user_function_start future", |c| Calls {
user_function_start: Some(future(c)),
..Calls::default()
}),
("on_user_function_end", |c| Calls {
user_function_end: Some(work(c)),
..Calls::default()
}),
("on_user_function_end future", |c| Calls {
user_function_end: Some(future(c)),
..Calls::default()
}),
("on_operation_start", |c| Calls {
operation_start: Some(work(c)),
..Calls::default()
}),
("on_operation_start future", |c| Calls {
operation_start: Some(future(c)),
..Calls::default()
}),
("on_invocation_start", |c| Calls {
invocation_start: Some(c),
..Calls::default()
}),
("name", |c| Calls {
name: Some(c),
..Calls::default()
}),
("error Display", |c| Calls {
operation_start: Some(Arc::new(move || {
Err(Box::new(CallsOnDisplay(Arc::clone(&c))))
})),
..Calls::default()
}),
("error destructor", |c| Calls {
operation_start: Some(Arc::new(move || Err(Box::new(CallsOnDrop(Arc::clone(&c)))))),
..Calls::default()
}),
("panic payload destructor", |c| Calls {
operation_start: Some(Arc::new(move || {
std::panic::panic_any(CallsOnDrop(Arc::clone(&c)))
})),
..Calls::default()
}),
];
let calls_wait = |ctx: &crate::DurableContext| -> Call {
let ctx = ctx.clone();
Arc::new(move || drop(ctx.wait(Duration::from_secs(1))))
};
for (case, calls) in cases {
let ctx = crate::DurableContext::test_context();
let plugins = calls_plugins(calls(calls_wait(&ctx)));
exercise(&plugins).await;
assert_context_usage_error(&ctx, case);
}
let ctx = crate::DurableContext::test_context();
let factory: Arc<dyn PluginFactory> = Arc::new(CallingFactory(calls_wait(&ctx)));
drop(create_plugins(
&[("CallingFactory", factory)],
None,
crate::plugin::tests_support_info,
));
assert_context_usage_error(&ctx, "factory");
let slot: Arc<Mutex<Option<crate::DurableContext>>> = Arc::default();
let end_slot = Arc::clone(&slot);
let plugin = crate::plugin::test_support::Hooks::default().on_end(move |_| {
if let Some(ctx) = end_slot.lock().unwrap().as_ref() {
drop(ctx.wait(Duration::from_secs(1)));
}
Box::pin(std::future::ready(Ok(())))
});
let handler_slot = Arc::clone(&slot);
let (service, event) =
crate::plugin::test_support::service(vec![plugin], 0, move |_event, ctx| {
*handler_slot.lock().unwrap() = Some(ctx);
async { Ok(serde_json::json!("done")) }
});
let response = service(event).await.expect("the invocation responds");
assert_eq!(
(
response.0.get("Status").and_then(serde_json::Value::as_str),
response.0.get("Result").and_then(serde_json::Value::as_str),
),
(Some("SUCCEEDED"), Some("\"done\"")),
"{}",
response.0
);
let ctx = slot.lock().unwrap().take().expect("the handler ran");
assert_context_usage_error(&ctx, "on_invocation_end");
}
#[tokio::test]
async fn unguarded_plugin_task_claim_is_refused() {
let ctx = crate::DurableContext::test_context();
let task_ctx = ctx.clone();
crate::task::spawn_plugin_work(async move {
drop(task_ctx.step(|_| async { Ok(1_u32) }));
})
.expect("a runtime is running")
.await
.expect("the task finished");
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert_eq!(
fatal.error_message,
crate::context::foreign_caller_message("step")
);
let fresh = crate::DurableContext::test_context();
assert_ne!(
ctx.claim_operation_id("step").positional(),
fresh.claim_operation_id("step").positional()
);
let ctx = crate::DurableContext::test_context();
let hook_ctx = ctx.clone();
let plugins = calls_plugins(Calls {
operation_start: Some(Arc::new(move || {
let ctx = hook_ctx.clone();
Ok(Some(Box::pin(async move {
drop(ctx.step(|_| async { Ok(1_u32) }));
Ok(())
})))
})),
..Calls::default()
});
let op = OperationInfo::for_test("s");
plugins.dispatch_work("on_operation_start", |p| p.on_operation_start(&op));
settle_work(&plugins).await;
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert!(
fatal.error_message.contains("inside plugin code"),
"{}",
fatal.error_message
);
}
#[test]
fn body_panic_wins_over_wrap_and_subscriber_panics() {
use lambda_runtime::tracing::subscriber::layer::SubscriberExt as _;
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::PanicAfter), ("p1", Mode::Normal)], &log);
let mut run = std::pin::pin!(step_runner(&plugins, || PanicsWhenPolled("A")));
let subscriber = lambda_runtime::tracing::subscriber::registry().with(PanickingLayer);
let caught = {
let _guard = test_tracing::install_test_subscriber(subscriber);
catch_unwind(AssertUnwindSafe(|| futures_poll_once(run.as_mut())))
};
let payload = caught.expect_err("the body panicked");
assert_eq!(payload.downcast_ref::<&str>(), Some(&"A"));
assert_eq!(
take(&log),
[
"start p0:s in []",
"enter p0:s",
"start p1:s in [\"p0:s\"]",
"enter p1:s",
"exit p1:s",
"exit p0:s",
]
);
}
#[test]
fn ready_body_destructor_panic_resumes_after_wraps() {
#[derive(Debug)]
struct PanicsOnDrop(Option<&'static str>);
impl Drop for PanicsOnDrop {
#[expect(clippy::panic, reason = "the test needs destructors that panic")]
fn drop(&mut self) {
if let Some(message) = self.0 {
std::panic::panic_any(message);
}
}
}
struct ReadyBody {
_guard: PanicsOnDrop,
output: Option<&'static str>,
}
impl Future for ReadyBody {
type Output = Result<PanicsOnDrop, BoxError>;
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
Poll::Ready(Ok(PanicsOnDrop(self.output)))
}
}
for output in [None, Some("output")] {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal)], &log);
let mut run = std::pin::pin!(step_runner(&plugins, move || ReadyBody {
_guard: PanicsOnDrop(Some("drop")),
output,
}));
let payload = catch_unwind(AssertUnwindSafe(|| futures_poll_once(run.as_mut())))
.expect_err("the destructor panic resumes");
assert_eq!(payload.downcast_ref::<&str>(), Some(&"drop"));
assert_eq!(
take(&log),
["start p0:s in []", "enter p0:s", "exit p0:s"],
"{output:?}"
);
}
}
#[tokio::test]
async fn next_restores_the_active_body_from_before_the_wraps() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal), ("p1", Mode::Normal)], &log);
let frames = frames(&plugins, &["c"]);
let unset = frames.enter(current_body);
assert!(matches!(unset, ActiveBody::Unset), "{unset:?}");
let outer = run_body(ActiveBody::Step("9".to_owned()), async {
frames.enter(current_body)
})
.await;
assert!(
matches!(outer, ActiveBody::Step(ref id) if id == "9"),
"{outer:?}"
);
}
#[tokio::test]
async fn a_dropped_body_is_dropped_inside_its_wraps() {
struct OnDrop(Arc<Mutex<Vec<String>>>);
impl Drop for OnDrop {
fn drop(&mut self) {
self.0
.lock()
.unwrap()
.push(format!("drop in {:?}", active()));
}
}
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal), ("p1", Mode::Normal)], &log);
let guard_log = Arc::clone(&log);
let mut run = Box::pin(run_user_function(
Some(Arc::clone(&plugins)),
info("c"),
UserBody::Step("c".to_owned()),
tracing::Span::none(),
move || async move {
let _guard = OnDrop(guard_log);
std::future::pending::<()>().await;
Ok::<u32, BoxError>(0)
},
|_| WireError::new(None::<String>, None::<String>),
));
for _ in 0..8 {
assert!(
inv(&plugins)
.enter(|| futures_poll_once(run.as_mut()))
.is_pending()
);
tokio::task::yield_now().await;
}
inv(&plugins).enter(|| drop(run));
let log = take(&log);
assert!(
log.contains(&"drop in [\"p0:inv\", \"p1:inv\", \"p0:c\", \"p1:c\"]".to_owned()),
"{log:?}"
);
assert!(!log.iter().any(|l| l.starts_with("end")), "{log:?}");
}
fn futures_poll_once<F: Future>(fut: Pin<&mut F>) -> Poll<F::Output> {
let waker = std::task::Waker::noop();
let mut cx = Context::from_waker(waker);
fut.poll(&mut cx)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_task_runs_inside_its_creators_frames_at_every_poll() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal), ("p1", Mode::Normal)], &log);
let frames = frames(&plugins, &["c"]);
let coordinator = crate::coordinator::InvocationCoordinator::new();
let ownership = Arc::new(crate::driver::TaskOwnership::new_current());
let task = crate::task::spawn_tracked(
crate::task::TaskContext::new(frames, tracing::Span::none()),
&coordinator,
&ownership,
async {
let mut seen = Vec::new();
for _ in 0..16 {
seen.push((std::thread::current().id(), active()));
tokio::task::yield_now().await;
}
seen
},
)
.expect("a runtime is running");
let seen = task.await.into_value();
assert!(
seen.iter()
.all(|(_, a)| a == &["p0:inv", "p1:inv", "p0:c", "p1:c"]),
"{seen:?}"
);
assert!(active().is_empty(), "the spawning task has no frames");
}
#[tokio::test]
async fn an_aborted_task_drops_its_future_inside_its_span() {
use lambda_runtime::tracing::subscriber::registry;
struct OnDrop(Arc<Mutex<Vec<Option<tracing::span::Id>>>>);
impl Drop for OnDrop {
fn drop(&mut self) {
self.0.lock().unwrap().push(tracing::Span::current().id());
}
}
let _default = test_tracing::install_test_subscriber(registry());
let span = tracing::info_span!("creator");
let seen = Arc::new(Mutex::new(Vec::new()));
let guard = OnDrop(Arc::clone(&seen));
let coordinator = crate::coordinator::InvocationCoordinator::new();
let ownership = Arc::new(crate::driver::TaskOwnership::new_current());
let task = crate::task::spawn_tracked(
crate::task::TaskContext::new(Frames::none(), span.clone()),
&coordinator,
&ownership,
async move {
let _guard = guard;
std::future::pending::<()>().await;
},
)
.expect("a runtime is running");
tokio::task::yield_now().await;
task.abort_handle().expect("the task started").abort();
drop(task);
for _ in 0..8 {
if !seen.lock().unwrap().is_empty() {
break;
}
tokio::task::yield_now().await;
}
assert!(span.id().is_some(), "the subscriber enables the span");
assert_eq!(*seen.lock().unwrap(), [span.id()]);
}
#[tokio::test]
async fn a_spawned_handle_keeps_its_creators_task_context() {
let log = Arc::new(Mutex::new(Vec::new()));
let plugins = plugins(&[("p0", Mode::Normal)], &log);
let ctx = crate::DurableContext::test_context();
let handle = frames(&plugins, &["c"]).enter(|| {
ctx.step(|_| async { std::future::pending::<Result<u32, BoxError>>().await })
.name("s")
.spawn()
});
let seen = handle.task_context().frames().enter(active);
assert_eq!(seen, ["p0:inv", "p0:c"]);
}
struct PanickingLayer;
impl<S: tracing::Subscriber> lambda_runtime::tracing::subscriber::Layer<S> for PanickingLayer {
#[expect(clippy::panic, reason = "the test needs a subscriber that panics")]
fn on_event(
&self,
_event: &tracing::Event<'_>,
_ctx: lambda_runtime::tracing::subscriber::layer::Context<'_, S>,
) {
panic!("subscriber panicked");
}
}
#[derive(Clone, Default)]
struct SpanWrap(Arc<Mutex<Vec<String>>>);
impl SpanWrap {
fn seen(&self, site: &str) {
let thread = std::thread::current().id();
push(
&self.0,
format!("{site} in {:?} on {thread:?}", span_name()),
);
}
}
impl Plugin for SpanWrap {
fn wrap_user_function(&self, _info: &OperationInfo, next: &mut dyn FnMut()) {
self.seen("wrap");
tracing::info_span!("plugin_span").in_scope(next);
}
fn on_user_function_start(&self, _info: &OperationInfo) -> HookWork {
self.seen("start");
Ok(None)
}
fn on_user_function_end(
&self,
_info: &OperationInfo,
_outcome: Result<(), &WireError>,
) -> HookWork {
self.seen("end");
Ok(None)
}
}
impl PluginFactory for SpanWrap {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
fn supplied_span() -> tracing::Span {
crate::tracing_layer::operation_span("arn", "req", None, "s", None, 1, false)
}
fn span_runner<B, M>(plugin: &SpanWrap, body: M) -> impl Future<Output = Ran<()>> + use<B, M>
where
M: FnOnce() -> B,
B: Future<Output = Result<(), BoxError>>,
{
run_user_function(
Some(crate::plugin::test_support::plugins_of(plugin.clone())),
info("s"),
UserBody::Step("s".to_owned()),
supplied_span(),
body,
|_: &BoxError| WireError::new(None::<String>, None::<String>),
)
}
#[expect(
clippy::indexing_slicing,
clippy::panic,
reason = "test assertions on JSON records"
)]
fn ancestry(buffer: &Arc<Mutex<Vec<u8>>>, message: &str) -> Vec<String> {
let output = String::from_utf8_lossy(&buffer.lock().unwrap()).to_string();
let record = output
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.find(|v| field(v, "message") == message)
.unwrap_or_else(|| panic!("no record {message:?} in {output}"));
record["spans"]
.as_array()
.map(|spans| spans.iter().map(|s| field(s, "name").to_owned()).collect())
.unwrap_or_default()
}
#[test]
fn runner_enters_supplied_span_outside_own_chain() {
let plugin = SpanWrap::default();
let polled = plugin.clone();
let (buffer, thread) = std::thread::spawn(move || {
let (buffer, _guard) = capture();
let mut run = std::pin::pin!(span_runner(&polled, || async {
polls(1, || tracing::info!("body record")).await;
Ok(())
}));
assert!(futures_poll_once(run.as_mut()).is_pending());
assert!(futures_poll_once(run.as_mut()).is_ready());
(buffer, std::thread::current().id())
})
.join()
.expect("the poller returns");
assert_eq!(
ancestry(&buffer, "body record"),
["durable_operation", "plugin_span"]
);
let on = format!("on {thread:?}");
assert_eq!(
take(&plugin.0),
[
format!("start in Some(\"durable_operation\") {on}"),
format!("wrap in Some(\"durable_operation\") {on}"),
format!("wrap in Some(\"durable_operation\") {on}"),
format!("end in Some(\"durable_operation\") {on}"),
]
);
}
#[test]
fn runner_drops_completed_and_pending_bodies_inside_own_chain() {
struct LogsOnDrop(&'static str);
impl Drop for LogsOnDrop {
fn drop(&mut self) {
tracing::info!("{}", self.0);
}
}
let plugin = SpanWrap::default();
let (buffer, _guard) = capture();
let mut completed = std::pin::pin!(span_runner(&plugin, || {
let guard = LogsOnDrop("completed body dropped");
async move {
let _guard = guard;
Ok(())
}
}));
assert!(futures_poll_once(completed.as_mut()).is_ready());
let mut pending = Box::pin(span_runner(&plugin, || {
let guard = LogsOnDrop("pending body dropped");
async move {
let _guard = guard;
std::future::pending::<Result<(), BoxError>>().await
}
}));
assert!(futures_poll_once(pending.as_mut()).is_pending());
drop(pending);
for message in ["completed body dropped", "pending body dropped"] {
assert_eq!(
ancestry(&buffer, message),
["durable_operation", "plugin_span"],
"{message}"
);
}
}
thread_local! {
static OPERATION_EXIT_ARMED: Cell<bool> = const { Cell::new(false) };
}
fn arm_operation_exit() {
OPERATION_EXIT_ARMED.with(|armed| armed.set(true));
}
struct ExitPanics;
impl<S> lambda_runtime::tracing::subscriber::Layer<S> for ExitPanics
where
S: tracing::Subscriber
+ for<'l> lambda_runtime::tracing::subscriber::registry::LookupSpan<'l>,
{
fn on_exit(
&self,
id: &tracing::span::Id,
ctx: lambda_runtime::tracing::subscriber::layer::Context<'_, S>,
) {
let operation = ctx
.span(id)
.is_some_and(|span| span.name() == "durable_operation");
let fire = operation && OPERATION_EXIT_ARMED.with(|armed| armed.replace(false));
assert!(!fire, "exit boom");
}
}
fn panic_message<T>(result: std::thread::Result<T>) -> Option<&'static str> {
let payload = result.err()?;
payload.downcast_ref::<&'static str>().copied()
}
#[test]
#[expect(
clippy::panic,
reason = "the test needs a body and a destructor that panic"
)]
fn runner_keeps_the_first_panic_over_a_supplied_span_exit_panic() {
use lambda_runtime::tracing::subscriber::layer::SubscriberExt as _;
struct ArmsThenPanicsOnDrop;
impl Drop for ArmsThenPanicsOnDrop {
fn drop(&mut self) {
arm_operation_exit();
panic!("drop boom");
}
}
let _guard = test_tracing::install_test_subscriber(
lambda_runtime::tracing::subscriber::registry().with(ExitPanics),
);
let plugin = SpanWrap::default();
let mut polled = Box::pin(span_runner(&plugin, || async {
arm_operation_exit();
panic!("body boom");
}));
let poll = catch_unwind(AssertUnwindSafe(|| futures_poll_once(polled.as_mut())));
assert_eq!(panic_message(poll), Some("body boom"));
assert!(!OPERATION_EXIT_ARMED.with(Cell::get), "the exit fired");
let mut pending = Box::pin(span_runner(&plugin, || {
let guard = ArmsThenPanicsOnDrop;
async move {
let _guard = guard;
std::future::pending::<Result<(), BoxError>>().await
}
}));
assert!(futures_poll_once(pending.as_mut()).is_pending());
let dropped = catch_unwind(AssertUnwindSafe(move || drop(pending)));
assert_eq!(panic_message(dropped), Some("drop boom"));
assert!(!OPERATION_EXIT_ARMED.with(Cell::get), "the exit fired");
let mut ready = Box::pin(span_runner(&plugin, || async {
arm_operation_exit();
Ok(())
}));
let poll = catch_unwind(AssertUnwindSafe(|| futures_poll_once(ready.as_mut())));
assert_eq!(panic_message(poll), Some("exit boom"));
}
thread_local! {
static OPERATION_CLOSE_ARMED: Cell<bool> = const { Cell::new(false) };
}
thread_local! {
static CLOSE_PAYLOAD_DROP_PANICS: Cell<bool> = const { Cell::new(false) };
}
fn arm_operation_close() {
OPERATION_CLOSE_ARMED.with(|armed| armed.set(true));
}
struct ClosePayloadDropPanics;
impl Drop for ClosePayloadDropPanics {
#[expect(
clippy::panic,
reason = "the test needs a payload whose destructor panics"
)]
fn drop(&mut self) {
panic!("payload drop boom");
}
}
struct ClosePanics;
impl<S> lambda_runtime::tracing::subscriber::Layer<S> for ClosePanics
where
S: tracing::Subscriber
+ for<'l> lambda_runtime::tracing::subscriber::registry::LookupSpan<'l>,
{
fn on_close(
&self,
id: tracing::span::Id,
ctx: lambda_runtime::tracing::subscriber::layer::Context<'_, S>,
) {
let operation = ctx
.span(&id)
.is_some_and(|span| span.name() == "durable_operation");
let fire = operation && OPERATION_CLOSE_ARMED.with(|armed| armed.replace(false));
if fire && CLOSE_PAYLOAD_DROP_PANICS.with(|set| set.replace(false)) {
#[expect(
clippy::panic,
reason = "the test needs a close panic whose payload's destructor panics"
)]
std::panic::panic_any(ClosePayloadDropPanics);
}
assert!(!fire, "close boom");
}
}
#[test]
#[expect(clippy::panic, reason = "the test needs a destructor that panics")]
fn runner_drop_keeps_the_first_panic_over_a_span_close_panic() {
use lambda_runtime::tracing::subscriber::layer::SubscriberExt as _;
struct ArmsCloseThenPanicsOnDrop;
impl Drop for ArmsCloseThenPanicsOnDrop {
fn drop(&mut self) {
arm_operation_close();
panic!("drop boom");
}
}
let _guard = test_tracing::install_test_subscriber(
lambda_runtime::tracing::subscriber::registry().with(ClosePanics),
);
let plugin = SpanWrap::default();
let mut both = Box::pin(span_runner(&plugin, || {
let guard = ArmsCloseThenPanicsOnDrop;
async move {
let _guard = guard;
std::future::pending::<Result<(), BoxError>>().await
}
}));
assert!(futures_poll_once(both.as_mut()).is_pending());
let dropped = catch_unwind(AssertUnwindSafe(move || drop(both)));
assert_eq!(panic_message(dropped), Some("drop boom"));
assert!(
!OPERATION_CLOSE_ARMED.with(Cell::get),
"the close did not run"
);
let mut pending = Box::pin(span_runner(&plugin, || async {
std::future::pending::<Result<(), BoxError>>().await
}));
assert!(futures_poll_once(pending.as_mut()).is_pending());
arm_operation_close();
let dropped = catch_unwind(AssertUnwindSafe(move || drop(pending)));
assert_eq!(panic_message(dropped), Some("close boom"));
let mut ready = Box::pin(span_runner(&plugin, || async { Ok(()) }));
assert!(futures_poll_once(ready.as_mut()).is_ready());
arm_operation_close();
let dropped = catch_unwind(AssertUnwindSafe(move || drop(ready)));
assert_eq!(panic_message(dropped), Some("close boom"));
assert!(
!OPERATION_CLOSE_ARMED.with(Cell::get),
"the close did not run"
);
}
#[test]
#[expect(clippy::panic, reason = "the test needs a destructor that panics")]
fn runner_drop_contains_a_close_panic_payload_whose_drop_panics() {
use lambda_runtime::tracing::subscriber::layer::SubscriberExt as _;
struct ArmsPanickingCloseThenPanicsOnDrop;
impl Drop for ArmsPanickingCloseThenPanicsOnDrop {
fn drop(&mut self) {
CLOSE_PAYLOAD_DROP_PANICS.with(|set| set.set(true));
arm_operation_close();
panic!("drop boom");
}
}
let _guard = test_tracing::install_test_subscriber(
lambda_runtime::tracing::subscriber::registry().with(ClosePanics),
);
let plugin = SpanWrap::default();
let mut pending = Box::pin(span_runner(&plugin, || {
let guard = ArmsPanickingCloseThenPanicsOnDrop;
async move {
let _guard = guard;
std::future::pending::<Result<(), BoxError>>().await
}
}));
assert!(futures_poll_once(pending.as_mut()).is_pending());
let dropped = catch_unwind(AssertUnwindSafe(move || drop(pending)));
assert_eq!(panic_message(dropped), Some("drop boom"));
assert!(
!OPERATION_CLOSE_ARMED.with(Cell::get),
"the close did not run"
);
assert!(
!CLOSE_PAYLOAD_DROP_PANICS.with(Cell::get),
"the close did not panic with the payload"
);
}
}