use std::collections::BTreeMap;
use std::fmt;
use std::future::Future;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::pin::Pin;
use std::sync::{Arc, Mutex, Weak};
use std::task::{Context, Poll};
use std::time::{Duration, SystemTime};
use tokio::task::JoinHandle;
use crate::coordinator::InvocationCoordinator;
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
use crate::{BoxError, InvocationStatus, OperationKind, OperationStatus, WireError};
#[cfg(test)]
pub(crate) mod recorder;
mod scope;
pub(crate) use scope::{
Frames, Ran, UserBody, never_fails, run_user_function, run_user_function_at,
};
pub(crate) const TARGET: &str = "aws_durable_execution_sdk::plugin";
pub type HookFuture<'a> = Pin<Box<dyn Future<Output = Result<(), BoxError>> + Send + 'a>>;
pub type HookWork = Result<Option<HookFuture<'static>>, BoxError>;
pub trait PluginFactory: Send + Sync + 'static {
fn create_plugin(&self, info: &InvocationInfo) -> Result<Option<Box<dyn Plugin>>, BoxError>;
fn exclusive_group(&self) -> Option<&str> {
None
}
}
pub trait Plugin: Send + Sync {
fn name(&self) -> &'static str {
std::any::type_name::<Self>()
}
fn on_invocation_start<'a>(&'a self, _info: &'a InvocationInfo) -> HookFuture<'a> {
Box::pin(std::future::ready(Ok(())))
}
fn on_invocation_end<'a>(&'a self, _info: &'a InvocationEndInfo) -> HookFuture<'a> {
Box::pin(std::future::ready(Ok(())))
}
fn on_operation_start(&self, _info: &OperationInfo) -> HookWork {
Ok(None)
}
fn on_operation_end(&self, _info: &OperationInfo) -> HookWork {
Ok(None)
}
fn on_attempt_start(&self, _info: &AttemptInfo) -> HookWork {
Ok(None)
}
fn on_attempt_end(&self, _info: &AttemptEndInfo) -> HookWork {
Ok(None)
}
fn on_operation_change(&self, _info: &OperationChangeInfo) -> HookWork {
Ok(None)
}
fn wrap_invocation(&self, _info: &InvocationInfo, next: &mut dyn FnMut()) {
next();
}
fn wrap_user_function(&self, _info: &OperationInfo, next: &mut dyn FnMut()) {
next();
}
fn on_user_function_start(&self, _info: &OperationInfo) -> HookWork {
Ok(None)
}
fn on_user_function_end(
&self,
_info: &OperationInfo,
_outcome: Result<(), &WireError>,
) -> HookWork {
Ok(None)
}
fn propagation_trace_id(&self, _input: &PropagationInput) -> Result<Option<String>, BoxError> {
Ok(None)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum AttemptOutcome {
Succeeded,
Failed,
}
impl AttemptOutcome {
#[must_use]
pub fn wire_str(self) -> &'static str {
match self {
Self::Succeeded => "SUCCEEDED",
Self::Failed => "FAILED",
}
}
}
impl fmt::Display for AttemptOutcome {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Succeeded => "Succeeded",
Self::Failed => "Failed",
})
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct InvocationInfo {
request_id: String,
execution_arn: String,
execution_input: String,
operations: BTreeMap<String, OperationInfo>,
updated_operations: Vec<OperationInfo>,
execution_start_timestamp: Option<SystemTime>,
is_first_invocation: bool,
x_ray_trace_id: Option<String>,
}
impl InvocationInfo {
#[expect(clippy::too_many_arguments, reason = "one argument per field")]
pub(crate) fn new(
request_id: String,
execution_arn: String,
execution_input: String,
operations: BTreeMap<String, OperationInfo>,
updated_ids: Vec<String>,
execution_start_timestamp: Option<SystemTime>,
is_first_invocation: bool,
x_ray_trace_id: Option<String>,
) -> Self {
let mut kept: Vec<String> = Vec::new();
for id in updated_ids {
if operations.contains_key(&id) && !kept.contains(&id) {
kept.push(id);
}
}
let updated_operations = kept
.iter()
.filter_map(|id| operations.get(id).cloned())
.collect();
Self {
request_id,
execution_arn,
execution_input,
operations,
updated_operations,
execution_start_timestamp,
is_first_invocation,
x_ray_trace_id,
}
}
pub fn request_id(&self) -> &str {
&self.request_id
}
pub fn execution_arn(&self) -> &str {
&self.execution_arn
}
pub fn execution_input(&self) -> &str {
&self.execution_input
}
pub fn operations(&self) -> &BTreeMap<String, OperationInfo> {
&self.operations
}
pub fn updated_operations(&self) -> &[OperationInfo] {
&self.updated_operations
}
pub fn execution_start_timestamp(&self) -> Option<SystemTime> {
self.execution_start_timestamp
}
pub fn is_first_invocation(&self) -> bool {
self.is_first_invocation
}
pub fn x_ray_trace_id(&self) -> Option<&str> {
self.x_ray_trace_id.as_deref()
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct InvocationEndInfo {
request_id: String,
execution_arn: String,
execution_input: String,
operations: BTreeMap<String, OperationInfo>,
execution_start_timestamp: Option<SystemTime>,
status: InvocationStatus,
execution_result: Option<String>,
execution_error: Option<WireError>,
}
impl InvocationEndInfo {
pub fn request_id(&self) -> &str {
&self.request_id
}
pub fn execution_arn(&self) -> &str {
&self.execution_arn
}
pub fn execution_input(&self) -> &str {
&self.execution_input
}
pub fn operations(&self) -> &BTreeMap<String, OperationInfo> {
&self.operations
}
pub fn execution_start_timestamp(&self) -> Option<SystemTime> {
self.execution_start_timestamp
}
pub fn status(&self) -> InvocationStatus {
self.status
}
pub fn execution_result(&self) -> Option<&str> {
self.execution_result.as_deref()
}
pub fn execution_error(&self) -> Option<&WireError> {
self.execution_error.as_ref()
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct OperationInfo {
id: String,
name: Option<String>,
kind: Option<OperationKind>,
type_str: Option<String>,
sub_type: Option<String>,
parent_id: Option<String>,
status: Option<OperationStatus>,
status_str: Option<String>,
start_timestamp: Option<SystemTime>,
end_timestamp: Option<SystemTime>,
result: Option<String>,
error: Option<WireError>,
attempt: Option<u32>,
is_replay: bool,
is_virtual: bool,
}
#[cfg(test)]
std::thread_local! {
static INFOS_FROM_RECORDS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
#[cfg(test)]
pub(crate) fn infos_from_records() -> usize {
INFOS_FROM_RECORDS.with(std::cell::Cell::get)
}
impl OperationInfo {
pub(crate) fn from_record(id: &str, record: &CheckpointRecord) -> Self {
#[cfg(test)]
INFOS_FROM_RECORDS.with(|count| count.set(count.get() + 1));
let (status, status_str) = status_fields(&record.status);
let error = wire_error_from_fields(
record.error_type.as_deref(),
record.error_message.as_deref(),
record.error_data.as_deref(),
record.stack_trace.as_deref(),
)
.or_else(|| {
wire_error_from_fields(
record.invoke_error_type.as_deref(),
record.invoke_error_message.as_deref(),
record.invoke_error_data.as_deref(),
record.invoke_stack_trace.as_deref(),
)
});
Self {
id: id.to_owned(),
name: record.op_name.clone(),
kind: record.op_type.as_deref().and_then(kind_from_wire),
type_str: record.op_type.clone(),
sub_type: record.sub_type.clone(),
parent_id: record.parent_id.clone(),
status,
status_str: Some(status_str),
start_timestamp: record.start_timestamp,
end_timestamp: record.end_timestamp,
result: record
.result
.clone()
.or_else(|| record.invoke_result.clone()),
error,
attempt: (record.attempt > 0).then_some(record.attempt),
is_replay: false,
is_virtual: false,
}
}
pub(crate) fn execution(
id: &str,
type_str: &str,
status: &CheckpointStatus,
parent_id: Option<String>,
start_timestamp: Option<SystemTime>,
end_timestamp: Option<SystemTime>,
) -> Option<Self> {
let kind = kind_from_wire(type_str);
if kind != Some(OperationKind::Execution) {
return None;
}
let (status, status_str) = status_fields(status);
Some(Self {
id: id.to_owned(),
name: None,
kind,
type_str: Some(type_str.to_owned()),
sub_type: None,
parent_id,
status,
status_str: Some(status_str),
start_timestamp,
end_timestamp,
result: None,
error: None,
attempt: None,
is_replay: false,
is_virtual: false,
})
}
pub fn id(&self) -> &str {
&self.id
}
pub fn name(&self) -> Option<&str> {
self.name.as_deref()
}
pub fn kind(&self) -> Option<OperationKind> {
self.kind
}
pub fn type_str(&self) -> Option<&str> {
self.kind
.map(OperationKind::wire_str)
.or(self.type_str.as_deref())
}
pub fn sub_type(&self) -> Option<&str> {
self.sub_type.as_deref()
}
pub fn parent_id(&self) -> Option<&str> {
self.parent_id.as_deref()
}
pub fn status(&self) -> Option<OperationStatus> {
self.status
}
pub fn status_str(&self) -> Option<&str> {
self.status
.map(OperationStatus::wire_str)
.or(self.status_str.as_deref())
}
pub fn start_timestamp(&self) -> Option<SystemTime> {
self.start_timestamp
}
pub fn end_timestamp(&self) -> Option<SystemTime> {
self.end_timestamp
}
pub fn result(&self) -> Option<&str> {
self.result.as_deref()
}
pub fn error(&self) -> Option<&WireError> {
self.error.as_ref()
}
pub fn attempt(&self) -> Option<u32> {
self.attempt
}
pub fn is_replay(&self) -> bool {
self.is_replay
}
pub fn is_virtual(&self) -> bool {
self.is_virtual
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct AttemptInfo {
operation: Arc<OperationInfo>,
attempt: u32,
}
impl AttemptInfo {
pub fn operation(&self) -> &OperationInfo {
&self.operation
}
pub fn attempt(&self) -> u32 {
self.attempt
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct AttemptEndInfo {
operation: OperationInfo,
attempt: u32,
outcome: AttemptOutcome,
error: Option<WireError>,
}
impl AttemptEndInfo {
pub fn operation(&self) -> &OperationInfo {
&self.operation
}
pub fn attempt(&self) -> u32 {
self.attempt
}
pub fn outcome(&self) -> AttemptOutcome {
self.outcome
}
pub fn error(&self) -> Option<&WireError> {
self.error.as_ref()
}
}
impl OperationInfo {
pub(crate) fn for_operation(
ctx: &crate::DurableContext,
wire_id: &str,
kind: OperationKind,
sub_type: &str,
name: Option<&str>,
) -> Self {
let mut info = ctx
.with_checkpoint_record_by_wire(wire_id, |record| Self::from_record(wire_id, record))
.unwrap_or_else(|| Self {
id: wire_id.to_owned(),
name: None,
kind: None,
type_str: None,
sub_type: None,
parent_id: None,
status: None,
status_str: None,
start_timestamp: None,
end_timestamp: None,
result: None,
error: None,
attempt: None,
is_replay: false,
is_virtual: false,
});
info.kind = info.kind.or(Some(kind));
if info.sub_type.is_none() {
info.sub_type = Some(sub_type.to_owned());
}
if info.name.is_none() {
info.name = name.map(str::to_owned);
}
if info.parent_id.is_none() {
info.parent_id = ctx.parent_wire_id().map(str::to_owned);
}
info
}
pub(crate) fn with_status(mut self, status: OperationStatus) -> Self {
self.status = Some(status);
self
}
pub(crate) fn with_result(mut self, result: Option<String>) -> Self {
self.result = result;
self
}
pub(crate) fn with_error(mut self, error: Option<WireError>) -> Self {
self.error = error;
self
}
pub(crate) fn with_name(mut self, name: Option<String>) -> Self {
self.name = name;
self
}
pub(crate) fn with_replay(mut self, is_replay: bool) -> Self {
self.is_replay = is_replay;
self
}
pub(crate) fn with_virtual(mut self, is_virtual: bool) -> Self {
self.is_virtual = is_virtual;
self
}
pub(crate) fn with_attempt(mut self, attempt: u32) -> Self {
self.attempt = Some(attempt);
self
}
pub(crate) fn with_start_timestamp(mut self, at: SystemTime) -> Self {
self.start_timestamp = Some(at);
self
}
pub(crate) fn with_end_timestamp(mut self, at: SystemTime) -> Self {
self.end_timestamp = Some(at);
self
}
}
impl AttemptInfo {
pub(crate) fn new(operation: Arc<OperationInfo>, attempt: u32) -> Self {
Self { operation, attempt }
}
}
impl AttemptEndInfo {
pub(crate) fn new(
operation: OperationInfo,
attempt: u32,
outcome: AttemptOutcome,
error: Option<WireError>,
) -> Self {
Self {
operation,
attempt,
outcome,
error,
}
}
}
pub(crate) fn with_observed_end(info: OperationInfo) -> OperationInfo {
if info.end_timestamp.is_some() {
return info;
}
info.with_end_timestamp(SystemTime::now())
}
pub(crate) fn attempt_started(
plugins: &InvocationPlugins,
operation: OperationInfo,
attempt: u32,
is_replay: bool,
) -> Arc<OperationInfo> {
let started = operation.with_status(OperationStatus::Started);
operation_started(plugins, &started.clone().with_replay(is_replay));
attempt_start_only(plugins, started, attempt)
}
pub(crate) fn attempt_start_only(
plugins: &InvocationPlugins,
operation: OperationInfo,
attempt: u32,
) -> Arc<OperationInfo> {
let started = operation
.with_status(OperationStatus::Started)
.with_attempt(attempt)
.with_start_timestamp(SystemTime::now());
let info = AttemptInfo::new(Arc::new(started), attempt);
plugins.dispatch_work("on_attempt_start", |plugin| plugin.on_attempt_start(&info));
Arc::clone(&info.operation)
}
pub(crate) fn attempt_ended(
plugins: &InvocationPlugins,
operation: OperationInfo,
end: AttemptEnd,
) {
let operation = with_observed_end(operation);
let ends_operation = end.ends_operation;
let info = AttemptEndInfo::new(operation, end.attempt, end.outcome, end.error);
plugins.dispatch_work("on_attempt_end", |plugin| plugin.on_attempt_end(&info));
if ends_operation {
let operation = info.operation;
plugins.dispatch_work("on_operation_end", |plugin| {
plugin.on_operation_end(&operation)
});
}
}
pub(crate) fn operation_started(plugins: &InvocationPlugins, info: &OperationInfo) {
plugins.dispatch_work("on_operation_start", |plugin| {
plugin.on_operation_start(info)
});
}
pub(crate) fn operation_ended(plugins: &InvocationPlugins, info: &OperationInfo) {
plugins.dispatch_work("on_operation_end", |plugin| plugin.on_operation_end(info));
}
pub(crate) struct AttemptEnd {
pub(crate) attempt: u32,
pub(crate) outcome: AttemptOutcome,
pub(crate) error: Option<WireError>,
pub(crate) ends_operation: bool,
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct OperationChangeInfo {
execution_arn: String,
updated_operations: Vec<OperationInfo>,
operations: BTreeMap<String, OperationInfo>,
}
impl OperationChangeInfo {
pub fn execution_arn(&self) -> &str {
&self.execution_arn
}
pub fn updated_operations(&self) -> &[OperationInfo] {
&self.updated_operations
}
pub fn operations(&self) -> &BTreeMap<String, OperationInfo> {
&self.operations
}
pub(crate) fn new(
execution_arn: String,
updated_operations: Vec<OperationInfo>,
operations: BTreeMap<String, OperationInfo>,
) -> Self {
Self {
execution_arn,
updated_operations,
operations,
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct PropagationInput {
execution_arn: String,
operation_id: String,
parent_operation_id: Option<String>,
target_function_name: String,
}
impl PropagationInput {
pub(crate) fn new(
execution_arn: String,
operation_id: String,
parent_operation_id: Option<String>,
target_function_name: String,
) -> Self {
Self {
execution_arn,
operation_id,
parent_operation_id,
target_function_name,
}
}
pub fn execution_arn(&self) -> &str {
&self.execution_arn
}
pub fn operation_id(&self) -> &str {
&self.operation_id
}
pub fn parent_operation_id(&self) -> Option<&str> {
self.parent_operation_id.as_deref()
}
pub fn target_function_name(&self) -> &str {
&self.target_function_name
}
}
fn kind_from_wire(op_type: &str) -> Option<OperationKind> {
let canonical = crate::context::canonical_op_type(op_type);
OperationKind::from_wire(&aws_sdk_lambda::types::OperationType::from(
canonical.as_str(),
))
}
fn status_fields(status: &CheckpointStatus) -> (Option<OperationStatus>, String) {
let typed = match status {
CheckpointStatus::Started => Some(OperationStatus::Started),
CheckpointStatus::Pending => Some(OperationStatus::Pending),
CheckpointStatus::Ready => Some(OperationStatus::Ready),
CheckpointStatus::Succeeded => Some(OperationStatus::Succeeded),
CheckpointStatus::Failed => Some(OperationStatus::Failed),
CheckpointStatus::Cancelled => Some(OperationStatus::Cancelled),
CheckpointStatus::TimedOut => Some(OperationStatus::TimedOut),
CheckpointStatus::Stopped => Some(OperationStatus::Stopped),
CheckpointStatus::Unknown(_) => None,
};
(typed, status.wire_str().to_owned())
}
fn wire_error_from_fields(
error_type: Option<&str>,
error_message: Option<&str>,
error_data: Option<&str>,
stack_trace: Option<&[String]>,
) -> Option<WireError> {
if error_type.is_none()
&& error_message.is_none()
&& error_data.is_none()
&& stack_trace.is_none()
{
return None;
}
Some(
WireError::new(error_type, error_message)
.with_error_data(error_data)
.with_stack_trace(stack_trace.map(<[String]>::to_vec).unwrap_or_default()),
)
}
pub(crate) fn operation_infos(log: &CheckpointLog) -> BTreeMap<String, OperationInfo> {
log.project_records(OperationInfo::from_record)
.into_iter()
.map(|info| (info.id.clone(), info))
.collect()
}
pub(crate) fn operations_map(
log: &CheckpointLog,
execution: Option<&OperationInfo>,
) -> BTreeMap<String, OperationInfo> {
let mut map = operation_infos(log);
if let Some(execution) = execution {
map.insert(execution.id.clone(), execution.clone());
}
map
}
type Payload = Box<dyn std::any::Any + Send + 'static>;
pub(crate) struct PluginSlot {
pub(crate) plugin: Arc<dyn Plugin>,
pub(crate) identity: String,
}
pub(crate) const PLUGIN_DRAIN_RESERVE: Duration = Duration::from_millis(500);
pub(crate) const PLUGIN_END_RESERVE: Duration = Duration::from_millis(50);
enum WorkState {
Open(Vec<JoinHandle<()>>),
Closed,
}
enum Undelivered {
Kept(Vec<Payload>),
Attached {
coordinator: Arc<InvocationCoordinator>,
late: Vec<Payload>,
},
Discarding,
}
impl Undelivered {
fn record(this: &Mutex<Self>, payload: Payload) {
let coordinator = {
let mut state = this
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match &mut *state {
Self::Kept(kept) => {
kept.push(payload);
return;
}
Self::Attached { coordinator, .. } => Arc::clone(coordinator),
Self::Discarding => {
drop(state);
discard_plugin_panic(payload);
return;
}
}
};
let Some(rejected) = coordinator.record_panic(payload) else {
return;
};
Self::keep_late(this, rejected);
}
fn keep_late(this: &Mutex<Self>, payload: Payload) {
let rejected = {
let mut state = this
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Self::Attached { late, .. } = &mut *state {
late.push(payload);
return;
}
payload
};
discard_plugin_panic(rejected);
}
fn discard_all(this: &Mutex<Self>) {
let state = std::mem::replace(
&mut *this
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
Self::Discarding,
);
if let Self::Kept(left) | Self::Attached { late: left, .. } = state {
for payload in left {
discard_plugin_panic(payload);
}
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum LatePanics {
Fail,
Discard,
}
pub(crate) struct InvocationPlugins {
info: InvocationInfo,
plugins: Vec<PluginSlot>,
execution: Mutex<Option<OperationInfo>>,
undelivered: Arc<Mutex<Undelivered>>,
work: Mutex<WorkState>,
change_queue: Mutex<std::collections::VecDeque<Vec<OperationInfo>>>,
draining: std::sync::atomic::AtomicBool,
changes_pushed: std::sync::atomic::AtomicU64,
delivery: Mutex<ChangeDelivery>,
#[cfg(test)]
after_release: Mutex<Option<AfterRelease>>,
}
#[cfg(test)]
type AfterRelease = Box<dyn FnMut(&InvocationPlugins) + Send>;
#[derive(Default)]
pub(crate) struct ChangeDelivery {
delivered: u64,
waiters: BTreeMap<u64, std::task::Waker>,
next_key: u64,
}
#[derive(Debug, Default)]
pub(crate) struct DeliveryWait {
target: Option<u64>,
key: Option<u64>,
}
std::thread_local! {
static DRAINING_HERE: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
struct DrainingHere(usize);
impl DrainingHere {
fn enter(plugins: &InvocationPlugins) -> Self {
let address = std::ptr::from_ref(plugins) as usize;
Self(DRAINING_HERE.with(|here| here.replace(address)))
}
}
impl Drop for DrainingHere {
fn drop(&mut self) {
DRAINING_HERE.with(|here| here.set(self.0));
}
}
impl InvocationPlugins {
#[expect(
clippy::expect_used,
reason = "every index comes from the range of the immutable plugin vector"
)]
pub(crate) fn slot(&self, index: usize) -> &PluginSlot {
self.plugins
.get(index)
.expect("a plugin index comes from the plugin vector's range")
}
pub(crate) fn info(&self) -> &InvocationInfo {
&self.info
}
pub(crate) fn is_updated(&self, id: &str) -> bool {
self.info
.updated_operations()
.iter()
.any(|operation| operation.id() == id)
}
pub(crate) fn replace_execution(&self, info: OperationInfo) -> bool {
let mut slot = self
.execution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let changed = slot
.as_ref()
.is_none_or(|held| held.status_str != info.status_str);
*slot = Some(info);
changed
}
pub(crate) fn push_operation_change(&self, updated: Vec<OperationInfo>) {
let mut queue = self
.change_queue
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
queue.push_back(updated);
self.changes_pushed
.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
}
pub(crate) fn drain_operation_changes(&self, log: &CheckpointLog) {
use std::sync::atomic::Ordering::{AcqRel, Acquire};
loop {
if self
.draining
.compare_exchange(false, true, AcqRel, Acquire)
.is_err()
{
return;
}
let here = DrainingHere::enter(self);
while let Some(entry) = self.pop_operation_change() {
let operations = {
let execution = self
.execution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
operations_map(log, execution.as_ref())
};
let info = OperationChangeInfo::new(
self.info().execution_arn().to_owned(),
entry,
operations,
);
self.dispatch_work("on_operation_change", |p| p.on_operation_change(&info));
drop(info);
self.finish_delivery();
}
drop(here);
self.draining
.store(false, std::sync::atomic::Ordering::Release);
#[cfg(test)]
self.run_after_release();
if self.change_queue_is_empty() {
return;
}
}
}
fn finish_delivery(&self) {
let waiters = {
let mut delivery = self
.delivery
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
delivery.delivered += 1;
std::mem::take(&mut delivery.waiters)
};
for waiter in waiters.into_values() {
waiter.wake();
}
}
pub(crate) fn poll_changes_delivered(
&self,
log: &CheckpointLog,
wait: &mut DeliveryWait,
cx: &mut Context<'_>,
) -> Poll<()> {
use std::sync::atomic::Ordering::Acquire;
let target = *wait
.target
.get_or_insert_with(|| self.changes_pushed.load(Acquire));
self.drain_operation_changes(log);
let mut delivery = self
.delivery
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if delivery.delivered >= target
|| DRAINING_HERE.with(std::cell::Cell::get) == std::ptr::from_ref(self) as usize
{
return Poll::Ready(());
}
let key = *wait.key.get_or_insert_with(|| {
let key = delivery.next_key;
delivery.next_key += 1;
key
});
match delivery.waiters.entry(key) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
if !entry.get().will_wake(cx.waker()) {
entry.insert(cx.waker().clone());
}
}
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(cx.waker().clone());
}
}
Poll::Pending
}
pub(crate) fn cancel_wait(&self, wait: &mut DeliveryWait) {
if let Some(key) = wait.key.take() {
self.delivery
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.waiters
.remove(&key);
}
}
pub(crate) async fn changes_delivered(&self, log: &CheckpointLog) {
struct Guard<'a> {
plugins: &'a InvocationPlugins,
wait: DeliveryWait,
}
impl Drop for Guard<'_> {
fn drop(&mut self) {
self.plugins.cancel_wait(&mut self.wait);
}
}
let mut guard = Guard {
plugins: self,
wait: DeliveryWait::default(),
};
std::future::poll_fn(|cx| {
guard
.plugins
.poll_changes_delivered(log, &mut guard.wait, cx)
})
.await;
}
#[cfg(test)]
pub(crate) fn delivery_waiter_count(&self) -> usize {
self.delivery
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.waiters
.len()
}
#[cfg(test)]
fn run_after_release(&self) {
let hook = self
.after_release
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
if let Some(mut hook) = hook {
hook(self);
}
}
#[cfg(test)]
pub(crate) fn set_after_release(&self, hook: impl FnMut(&Self) + Send + 'static) {
*self
.after_release
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(hook));
}
fn pop_operation_change(&self) -> Option<Vec<OperationInfo>> {
self.change_queue
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.pop_front()
}
fn change_queue_is_empty(&self) -> bool {
self.change_queue
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty()
}
#[cfg(test)]
pub(crate) fn is_draining(&self) -> bool {
self.draining.load(std::sync::atomic::Ordering::Acquire)
}
#[cfg(test)]
pub(crate) fn take_error_drop_panics(&self) -> Vec<Payload> {
match &mut *self
.undelivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
{
Undelivered::Kept(kept) => std::mem::take(kept),
Undelivered::Attached { .. } | Undelivered::Discarding => Vec::new(),
}
}
pub(crate) fn attach(&self, coordinator: &Arc<InvocationCoordinator>) {
let kept = std::mem::replace(
&mut *self
.undelivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
Undelivered::Attached {
coordinator: Arc::clone(coordinator),
late: Vec::new(),
},
);
if let Undelivered::Kept(kept) = kept {
for payload in kept {
if let Some(rejected) = coordinator.record_panic(payload) {
discard_plugin_panic(rejected);
}
}
}
}
pub(crate) fn shut_down(&self) {
if let WorkState::Open(list) = std::mem::replace(&mut *self.lock_work(), WorkState::Closed)
{
for handle in list {
handle.abort();
}
}
Undelivered::discard_all(&self.undelivered);
}
pub(crate) async fn invocation_start(&self) {
let info = self.info();
JoinHooks::new(self.runs("on_invocation_start", |plugin| {
plugin.on_invocation_start(info)
}))
.await;
}
pub(crate) async fn invocation_end(
&self,
log: &CheckpointLog,
end: crate::InvocationEnd,
remaining: Option<Duration>,
late: LatePanics,
) -> Option<Payload> {
let now = tokio::time::Instant::now();
let by = |reserve: Duration| remaining.map(|left| now + left.saturating_sub(reserve));
self.drain(by(PLUGIN_DRAIN_RESERVE)).await;
let drained = self.take_late_panic(late);
let end = match &drained {
Some(payload) => crate::InvocationEnd::panicked(&**payload),
None => end,
};
let info = self.end_info(log, end);
let hooks = JoinHooks::new(self.runs("on_invocation_end", |plugin| {
plugin.on_invocation_end(&info)
}));
if !wait_until(by(PLUGIN_END_RESERVE), hooks).await {
warn_contained(|| {
tracing::warn!(
target: TARGET,
"plugin on_invocation_end hooks did not finish before the invocation deadline; \
the response returns without them"
);
});
}
let ended = self.take_late_panic(late);
match (drained, ended) {
(Some(first), later) => {
if let Some(later) = later {
discard_plugin_panic(later);
}
Some(first)
}
(None, ended) => ended,
}
}
fn take_late_panic(&self, policy: LatePanics) -> Option<Payload> {
let late = match &mut *self
.undelivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
{
Undelivered::Attached { late, .. } => std::mem::take(late),
Undelivered::Kept(_) | Undelivered::Discarding => Vec::new(),
};
let mut late = late.into_iter();
let first = match policy {
LatePanics::Fail => late.next(),
LatePanics::Discard => None,
};
for payload in late {
discard_plugin_panic(payload);
}
first
}
async fn drain(&self, drain_by: Option<tokio::time::Instant>) {
loop {
let mut joining = AbortOnDrop({
let mut state = self.lock_work();
match std::mem::replace(&mut *state, WorkState::Closed) {
WorkState::Open(list) if !list.is_empty() => {
*state = WorkState::Open(Vec::new());
list
}
_ => return,
}
});
let joined = wait_until(
drain_by,
std::future::poll_fn(|cx| {
joining
.0
.retain_mut(|handle| Pin::new(handle).poll(cx).is_pending());
if joining.0.is_empty() {
Poll::Ready(())
} else {
Poll::Pending
}
}),
)
.await;
if !joined {
if let WorkState::Open(rest) =
std::mem::replace(&mut *self.lock_work(), WorkState::Closed)
{
joining.0.extend(rest);
}
let abandoned = joining
.0
.iter()
.filter(|handle| !handle.is_finished())
.count();
for handle in &joining.0 {
handle.abort();
}
for handle in &mut joining.0 {
let _ = handle.await;
}
if abandoned > 0 {
warn_contained(|| {
tracing::warn!(
target: TARGET,
abandoned,
"plugin hook work did not finish before the drain deadline; it is aborted"
);
});
}
return;
}
}
}
pub(crate) fn dispatch_work(&self, hook: &'static str, call: impl Fn(&dyn Plugin) -> HookWork) {
self.dispatch_to(hook, 0..self.plugins.len(), call);
}
pub(crate) fn dispatch_to(
&self,
hook: &'static str,
indices: impl IntoIterator<Item = usize>,
call: impl Fn(&dyn Plugin) -> HookWork,
) -> Vec<usize> {
let mut ok = Vec::new();
for index in indices {
let slot = self.slot(index);
match call_plugin(|| call(slot.plugin.as_ref())) {
Ok(Ok(None)) => ok.push(index),
Ok(Ok(Some(work))) => {
ok.push(index);
self.start_work(index, hook, work);
}
Ok(Err(error)) => {
report_hook_error(&slot.identity, hook, &self.undelivered, error);
}
Err(payload) => log_hook_panic(&slot.identity, hook, payload),
}
}
ok
}
pub(crate) fn propagation_trace_id(&self, input: &PropagationInput) -> Option<String> {
let mut kept: Option<(String, &str)> = None;
let mut conflict_count: u64 = 0;
for slot in &self.plugins {
let value = match call_plugin(|| slot.plugin.propagation_trace_id(input)) {
Ok(Ok(Some(value))) => value,
Ok(Ok(None)) => continue,
Ok(Err(error)) => {
report_propagation_error(&slot.identity, &self.undelivered, error);
continue;
}
Err(payload) => {
warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %slot.identity,
hook = "propagation_trace_id",
"plugin propagation_trace_id panicked; the value is ignored"
);
});
discard_plugin_panic(payload);
continue;
}
};
if value.trim().is_empty() {
continue;
}
match &kept {
None => kept = Some((value, slot.identity.as_str())),
Some((first, _)) if *first == value => {}
Some((_, first_identity)) => {
conflict_count = conflict_count.saturating_add(1);
warn_contained(|| {
tracing::warn!(
target: TARGET,
field = "x_amzn_trace_id",
first_plugin = %first_identity,
later_plugin = %slot.identity,
conflict_count,
"conflicting propagation trace ids; keeping the first value"
);
});
}
}
}
kept.map(|(value, _)| value)
}
pub(crate) fn start_work(&self, slot: usize, hook: &'static str, work: HookFuture<'static>) {
#[expect(clippy::expect_used)]
let identity = &self
.plugins
.get(slot)
.expect("the slot names a plugin of this invocation")
.identity;
let mut state = self.lock_work();
let WorkState::Open(list) = &mut *state else {
drop(state);
drop_hook_future(identity, hook, work);
warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook,
"plugin hook work started after the drain closed; it is dropped"
);
});
return;
};
list.retain(|handle| !handle.is_finished());
let run = HookRun {
identity: identity.clone().into(),
hook,
future: Some(work),
undelivered: Arc::clone(&self.undelivered),
};
if let Ok(handle) = crate::task::spawn_plugin_work(run) {
list.push(handle);
} else {
drop(state);
warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook,
"plugin hook work could not be spawned; it is dropped"
);
});
}
}
fn end_info(&self, log: &CheckpointLog, end: crate::InvocationEnd) -> InvocationEndInfo {
let execution = self
.execution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
InvocationEndInfo {
request_id: self.info.request_id.clone(),
execution_arn: self.info.execution_arn.clone(),
execution_input: self.info.execution_input.clone(),
operations: operations_map(log, execution.as_ref()),
execution_start_timestamp: self.info.execution_start_timestamp,
status: end.status,
execution_result: end.result,
execution_error: end.error,
}
}
fn runs<'a>(
&'a self,
hook: &'static str,
call: impl Fn(&'a dyn Plugin) -> HookFuture<'a>,
) -> Vec<HookRun<'a>> {
self.plugins
.iter()
.map(|slot| {
let future = match call_plugin(|| call(slot.plugin.as_ref())) {
Ok(future) => Some(future),
Err(payload) => {
log_hook_panic(&slot.identity, hook, payload);
None
}
};
HookRun {
identity: slot.identity.as_str().into(),
hook,
future,
undelivered: Arc::clone(&self.undelivered),
}
})
.collect()
}
fn lock_work(&self) -> std::sync::MutexGuard<'_, WorkState> {
self.work
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
}
struct AbortOnDrop(Vec<JoinHandle<()>>);
impl Drop for AbortOnDrop {
fn drop(&mut self) {
for handle in &self.0 {
handle.abort();
}
}
}
async fn wait_until(
deadline: Option<tokio::time::Instant>,
future: impl Future<Output = ()>,
) -> bool {
if let Some(deadline) = deadline {
tokio::time::timeout_at(deadline, future).await.is_ok()
} else {
future.await;
true
}
}
pub(crate) fn call_plugin<R>(f: impl FnOnce() -> R) -> Result<R, Payload> {
crate::context::run_body_sync(crate::context::ActiveBody::Plugin, || {
catch_unwind(AssertUnwindSafe(f))
})
}
pub(crate) fn warn_contained(emit: impl FnOnce()) {
crate::context::run_body_sync(crate::context::ActiveBody::Plugin, || {
if let Err(payload) = catch_unwind(AssertUnwindSafe(emit)) {
crate::task::discard_panic(payload);
}
});
}
pub(crate) fn discard_plugin_panic(payload: Payload) {
crate::context::run_body_sync(crate::context::ActiveBody::Plugin, || {
crate::task::discard_panic(payload);
});
}
pub(crate) fn log_hook_panic(identity: &str, hook: &'static str, payload: Payload) {
warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook,
"plugin hook panicked; the panic is ignored"
);
});
discard_plugin_panic(payload);
}
fn drop_hook_future(identity: &str, hook: &'static str, future: HookFuture<'_>) {
if let Err(payload) = call_plugin(move || drop(future)) {
log_hook_panic(identity, hook, payload);
}
}
struct HookRun<'a> {
identity: std::borrow::Cow<'a, str>,
hook: &'static str,
future: Option<HookFuture<'a>>,
undelivered: Arc<Mutex<Undelivered>>,
}
fn report_hook_error(
identity: &str,
hook: &'static str,
undelivered: &Mutex<Undelivered>,
error: BoxError,
) {
match call_plugin(|| error.to_string()) {
Ok(message) => warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook,
error = %message,
"plugin hook failed; the error is ignored"
);
}),
Err(payload) => {
warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook,
"plugin hook failed, and its error panicked while it was formatted; \
the error is ignored"
);
});
discard_plugin_panic(payload);
}
}
if let Err(payload) = call_plugin(move || drop(error)) {
Undelivered::record(undelivered, payload);
}
}
fn report_propagation_error(identity: &str, undelivered: &Mutex<Undelivered>, error: BoxError) {
match call_plugin(|| error.to_string()) {
Ok(message) => warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook = "propagation_trace_id",
error = %message,
"plugin propagation_trace_id failed; the value is ignored"
);
}),
Err(payload) => {
warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook = "propagation_trace_id",
"plugin propagation_trace_id failed, and its error panicked while it was \
formatted; the value is ignored"
);
});
discard_plugin_panic(payload);
}
}
if let Err(payload) = call_plugin(move || drop(error)) {
Undelivered::record(undelivered, payload);
}
}
impl Future for HookRun<'_> {
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
let this = self.get_mut();
let Some(future) = this.future.as_mut() else {
return Poll::Ready(());
};
let polled = call_plugin(|| future.as_mut().poll(cx));
if matches!(polled, Ok(Poll::Pending)) {
return Poll::Pending;
}
if let Some(future) = this.future.take() {
drop_hook_future(&this.identity, this.hook, future);
}
match polled {
Ok(Poll::Ready(Err(error))) => {
report_hook_error(&this.identity, this.hook, &this.undelivered, error);
}
Err(payload) => log_hook_panic(&this.identity, this.hook, payload),
Ok(Poll::Ready(Ok(())) | Poll::Pending) => {}
}
Poll::Ready(())
}
}
impl Drop for HookRun<'_> {
fn drop(&mut self) {
if let Some(future) = self.future.take() {
drop_hook_future(&self.identity, self.hook, future);
}
}
}
struct JoinHooks<'a> {
runs: Vec<HookRun<'a>>,
}
impl<'a> JoinHooks<'a> {
fn new(runs: Vec<HookRun<'a>>) -> Self {
Self { runs }
}
}
impl Future for JoinHooks<'_> {
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
let mut finished = true;
for run in &mut self.get_mut().runs {
finished &= Pin::new(run).poll(cx).is_ready();
}
if finished {
Poll::Ready(())
} else {
Poll::Pending
}
}
}
impl Drop for InvocationPlugins {
fn drop(&mut self) {
self.shut_down();
for PluginSlot { plugin, identity } in std::mem::take(&mut self.plugins) {
if let Err(payload) = call_plugin(move || drop(plugin)) {
warn_contained(|| {
tracing::warn!(
target: TARGET,
plugin = %identity,
hook = "drop",
"plugin panicked while it was dropped; the panic is ignored"
);
});
discard_plugin_panic(payload);
}
}
}
}
pub(crate) struct ShutDownOnDrop(Weak<InvocationPlugins>);
impl ShutDownOnDrop {
pub(crate) fn new(plugins: &Arc<InvocationPlugins>) -> Self {
Self(Arc::downgrade(plugins))
}
}
impl Drop for ShutDownOnDrop {
fn drop(&mut self) {
if let Some(plugins) = self.0.upgrade() {
plugins.shut_down();
}
}
}
pub(crate) type Factories = [(&'static str, Arc<dyn PluginFactory>)];
pub(crate) fn create_plugins(
factories: &Factories,
execution: Option<OperationInfo>,
build_info: impl FnOnce() -> InvocationInfo,
) -> Option<Arc<InvocationPlugins>> {
if factories.is_empty() {
return None;
}
let info = build_info();
let mut error_drop_panics = Vec::new();
let plugins = create_slots(factories, &info, &mut error_drop_panics);
Some(Arc::new(InvocationPlugins {
info,
plugins,
execution: Mutex::new(execution),
undelivered: Arc::new(Mutex::new(Undelivered::Kept(error_drop_panics))),
work: Mutex::new(WorkState::Open(Vec::new())),
change_queue: Mutex::new(std::collections::VecDeque::new()),
draining: std::sync::atomic::AtomicBool::new(false),
changes_pushed: std::sync::atomic::AtomicU64::new(0),
delivery: Mutex::new(ChangeDelivery::default()),
#[cfg(test)]
after_release: Mutex::new(None),
}))
}
fn create_slots(
factories: &Factories,
info: &InvocationInfo,
error_drop_panics: &mut Vec<Payload>,
) -> Vec<PluginSlot> {
let mut slots = Vec::new();
for (index, (type_name, factory)) in factories.iter().enumerate() {
let factory_identity = format!("{type_name} (plugins[{index}])");
match call_plugin(|| factory.create_plugin(info)) {
Ok(Ok(Some(plugin))) => {
let plugin: Arc<dyn Plugin> = Arc::from(plugin);
let identity = match call_plugin(|| plugin.name()) {
Ok(name) => format!("{name} (plugins[{index}])"),
Err(payload) => {
let identity = format!("plugins[{index}]");
log_hook_panic(&identity, "name", payload);
identity
}
};
slots.push(PluginSlot { plugin, identity });
}
Ok(Ok(None)) => {}
Ok(Err(error)) => log_factory_error(&factory_identity, error, error_drop_panics),
Err(payload) => {
warn_contained(|| {
tracing::warn!(
target: TARGET,
factory = %factory_identity,
"plugin factory panicked; the plugin is skipped for this invocation"
);
});
discard_plugin_panic(payload);
}
}
}
slots
}
fn log_factory_error(
factory_identity: &str,
error: BoxError,
error_drop_panics: &mut Vec<Payload>,
) {
match call_plugin(|| error.to_string()) {
Ok(message) => warn_contained(|| {
tracing::warn!(
target: TARGET,
factory = %factory_identity,
error = %message,
"plugin factory failed; the plugin is skipped for this invocation"
);
}),
Err(payload) => {
warn_contained(|| {
tracing::warn!(
target: TARGET,
factory = %factory_identity,
"plugin factory failed, and its error panicked while it was formatted; \
the plugin is skipped for this invocation"
);
});
discard_plugin_panic(payload);
}
}
if let Err(payload) = call_plugin(move || drop(error)) {
error_drop_panics.push(payload);
}
}
#[cfg(test)]
pub(crate) fn tests_support_info() -> InvocationInfo {
InvocationInfo::new(
"r".to_owned(),
"arn".to_owned(),
"{}".to_owned(),
BTreeMap::new(),
Vec::new(),
None,
true,
None,
)
}
#[cfg(test)]
impl OperationInfo {
pub(crate) fn for_test(id: &str) -> Self {
Self {
id: id.to_owned(),
name: None,
kind: None,
type_str: None,
sub_type: None,
parent_id: None,
status: None,
status_str: None,
start_timestamp: None,
end_timestamp: None,
result: None,
error: None,
attempt: None,
is_replay: false,
is_virtual: false,
}
}
}
#[cfg(test)]
pub(crate) mod test_support {
use std::future::Future;
use std::sync::{Arc, Mutex};
use super::{HookFuture, InvocationEndInfo, InvocationInfo, Plugin, PluginFactory, TARGET};
use crate::{BoxError, DurableContext, InvocationPayload, Options};
type StartFn = dyn Fn(&InvocationInfo) -> HookFuture<'static> + Send + Sync;
type EndFn = dyn Fn(&InvocationEndInfo) -> HookFuture<'static> + Send + Sync;
#[derive(Clone, Default)]
pub(crate) struct Hooks {
pub(crate) name: Option<&'static str>,
pub(crate) start: Option<Arc<StartFn>>,
pub(crate) end: Option<Arc<EndFn>>,
pub(crate) starts: Arc<Mutex<Vec<InvocationInfo>>>,
pub(crate) ends: Arc<Mutex<Vec<InvocationEndInfo>>>,
}
impl Hooks {
pub(crate) fn on_start(
mut self,
hook: impl Fn(&InvocationInfo) -> HookFuture<'static> + Send + Sync + 'static,
) -> Self {
self.start = Some(Arc::new(hook));
self
}
pub(crate) fn on_end(
mut self,
hook: impl Fn(&InvocationEndInfo) -> HookFuture<'static> + Send + Sync + 'static,
) -> Self {
self.end = Some(Arc::new(hook));
self
}
pub(crate) fn named(mut self, name: &'static str) -> Self {
self.name = Some(name);
self
}
pub(crate) fn ends(&self) -> Vec<InvocationEndInfo> {
self.ends.lock().unwrap().clone()
}
}
impl Plugin for Hooks {
fn name(&self) -> &'static str {
self.name.unwrap_or("Hooks")
}
fn on_invocation_start<'a>(&'a self, info: &'a InvocationInfo) -> HookFuture<'a> {
self.starts.lock().unwrap().push(info.clone());
match &self.start {
Some(hook) => hook(info),
None => Box::pin(std::future::ready(Ok(()))),
}
}
fn on_invocation_end<'a>(&'a self, info: &'a InvocationEndInfo) -> HookFuture<'a> {
self.ends.lock().unwrap().push(info.clone());
match &self.end {
Some(hook) => hook(info),
None => Box::pin(std::future::ready(Ok(()))),
}
}
}
impl PluginFactory for Hooks {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
type ChangeFn = dyn Fn(&super::OperationChangeInfo) + Send + Sync;
#[derive(Clone, Default)]
pub(crate) struct Changes {
pub(crate) calls: Arc<Mutex<Vec<super::OperationChangeInfo>>>,
pub(crate) timeline: Arc<Mutex<Vec<String>>>,
pub(crate) on_change: Option<Arc<ChangeFn>>,
pub(crate) panics: bool,
}
impl Changes {
pub(crate) fn calls(&self) -> Vec<super::OperationChangeInfo> {
self.calls.lock().unwrap().clone()
}
pub(crate) fn timeline(&self) -> Vec<String> {
self.timeline.lock().unwrap().clone()
}
pub(crate) fn plugins(&self) -> Arc<super::InvocationPlugins> {
plugins_of(self.clone())
}
fn label(info: &super::OperationInfo) -> String {
info.name().unwrap_or(info.id()).to_owned()
}
pub(crate) fn updated_ids(&self) -> Vec<Vec<String>> {
self.calls()
.iter()
.map(|info| {
info.updated_operations()
.iter()
.map(|op| op.id().to_owned())
.collect()
})
.collect()
}
}
impl Plugin for Changes {
fn name(&self) -> &'static str {
"Changes"
}
fn on_operation_change(&self, info: &super::OperationChangeInfo) -> super::HookWork {
assert!(!self.panics, "change hook panicked");
self.calls.lock().unwrap().push(info.clone());
{
let mut timeline = self.timeline.lock().unwrap();
for op in info.updated_operations() {
timeline.push(format!(
"change:{}:{}",
Self::label(op),
op.status_str().unwrap_or_default()
));
}
}
if let Some(hook) = &self.on_change {
hook(info);
}
Ok(None)
}
fn on_operation_start(&self, info: &super::OperationInfo) -> super::HookWork {
let label = format!("start:{}", Self::label(info));
self.timeline.lock().unwrap().push(label);
Ok(None)
}
fn on_operation_end(&self, info: &super::OperationInfo) -> super::HookWork {
let label = format!("end:{}", Self::label(info));
self.timeline.lock().unwrap().push(label);
Ok(None)
}
fn on_invocation_end<'a>(&'a self, info: &'a InvocationEndInfo) -> HookFuture<'a> {
let label = format!("invocation_end:{:?}", info.status());
self.timeline.lock().unwrap().push(label);
Box::pin(std::future::ready(Ok(())))
}
}
impl PluginFactory for Changes {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
pub(crate) fn plugins_of_all(
factories: Vec<Arc<dyn PluginFactory>>,
) -> Arc<super::InvocationPlugins> {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> = factories
.into_iter()
.map(|factory| ("test::Plugin", factory))
.collect();
super::create_plugins(&factories, None, empty_info).expect("a factory is registered")
}
#[derive(Clone, Debug)]
pub(crate) struct OpRecord {
pub(crate) hook: &'static str,
pub(crate) name: Option<String>,
pub(crate) parent_id: Option<String>,
pub(crate) replay: bool,
pub(crate) is_virtual: bool,
pub(crate) status: Option<crate::OperationStatus>,
pub(crate) attempt: Option<u32>,
pub(crate) info_attempt: Option<u32>,
pub(crate) outcome: Option<super::AttemptOutcome>,
pub(crate) result: Option<String>,
pub(crate) error: Option<String>,
pub(crate) info_error: Option<crate::WireError>,
pub(crate) probe: String,
}
impl OpRecord {
pub(crate) fn shape(
&self,
) -> (
&'static str,
bool,
Option<u32>,
Option<super::AttemptOutcome>,
) {
(self.hook, self.replay, self.attempt, self.outcome)
}
}
type Probe = dyn Fn() -> String + Send + Sync;
#[derive(Clone, Default)]
pub(crate) struct OpHooks {
pub(crate) records: Arc<Mutex<Vec<OpRecord>>>,
pub(crate) probe: Option<Arc<Probe>>,
}
impl OpHooks {
pub(crate) fn probing(probe: impl Fn() -> String + Send + Sync + 'static) -> Self {
Self {
probe: Some(Arc::new(probe)),
..Self::default()
}
}
pub(crate) fn records(&self) -> Vec<OpRecord> {
self.records.lock().unwrap().clone()
}
pub(crate) fn of(&self, name: &str) -> Vec<OpRecord> {
self.records()
.into_iter()
.filter(|r| r.name.as_deref() == Some(name))
.collect()
}
fn push(
&self,
hook: &'static str,
info: &super::OperationInfo,
attempt: Option<u32>,
end: Option<&super::AttemptEndInfo>,
) {
let probe = self.probe.as_ref().map(|probe| probe()).unwrap_or_default();
self.records.lock().unwrap().push(OpRecord {
hook,
name: info.name().map(str::to_owned),
parent_id: info.parent_id().map(str::to_owned),
replay: info.is_replay(),
is_virtual: info.is_virtual(),
status: info.status(),
info_attempt: info.attempt(),
attempt,
outcome: end.map(super::AttemptEndInfo::outcome),
result: info.result().map(str::to_owned),
error: end
.and_then(super::AttemptEndInfo::error)
.and_then(crate::WireError::error_message)
.map(str::to_owned),
info_error: info.error().cloned(),
probe,
});
}
pub(crate) fn plugins(&self) -> Arc<super::InvocationPlugins> {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
vec![("test::OpHooks", Arc::new(self.clone()))];
super::create_plugins(&factories, None, empty_info).expect("one factory is registered")
}
}
impl Plugin for OpHooks {
fn on_operation_start(&self, info: &super::OperationInfo) -> super::HookWork {
self.push("start", info, None, None);
Ok(None)
}
fn on_operation_end(&self, info: &super::OperationInfo) -> super::HookWork {
self.push("end", info, None, None);
Ok(None)
}
fn on_attempt_start(&self, info: &super::AttemptInfo) -> super::HookWork {
self.push(
"attempt_start",
info.operation(),
Some(info.attempt()),
None,
);
Ok(None)
}
fn on_attempt_end(&self, info: &super::AttemptEndInfo) -> super::HookWork {
self.push(
"attempt_end",
info.operation(),
Some(info.attempt()),
Some(info),
);
Ok(None)
}
}
impl PluginFactory for OpHooks {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
type TraceFn =
dyn Fn(&super::PropagationInput) -> Result<Option<String>, BoxError> + Send + Sync;
#[derive(Clone)]
pub(crate) struct TraceHook {
answer: Arc<TraceFn>,
inputs: Arc<Mutex<Vec<super::PropagationInput>>>,
}
impl TraceHook {
pub(crate) fn new(
answer: impl Fn(&super::PropagationInput) -> Result<Option<String>, BoxError>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
answer: Arc::new(answer),
inputs: Arc::default(),
}
}
pub(crate) fn value(value: Option<&'static str>) -> Self {
Self::new(move |_| Ok(value.map(str::to_owned)))
}
pub(crate) fn inputs(&self) -> Vec<super::PropagationInput> {
self.inputs.lock().unwrap().clone()
}
}
impl Plugin for TraceHook {
fn name(&self) -> &'static str {
"TraceHook"
}
fn propagation_trace_id(
&self,
input: &super::PropagationInput,
) -> Result<Option<String>, BoxError> {
self.inputs.lock().unwrap().push(input.clone());
(self.answer)(input)
}
}
impl PluginFactory for TraceHook {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
pub(crate) fn trace_plugins(hooks: &[TraceHook]) -> Arc<super::InvocationPlugins> {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> = hooks
.iter()
.map(|hook| {
(
"test::TraceHook",
Arc::new(hook.clone()) as Arc<dyn PluginFactory>,
)
})
.collect();
super::create_plugins(&factories, None, empty_info).expect("a factory is registered")
}
fn empty_info() -> InvocationInfo {
InvocationInfo::new(
"r".to_owned(),
"arn".to_owned(),
"{}".to_owned(),
std::collections::BTreeMap::new(),
Vec::new(),
None,
true,
None,
)
}
pub(crate) fn plugins_of(
factory: impl PluginFactory + 'static,
) -> Arc<super::InvocationPlugins> {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
vec![("test::Plugin", Arc::new(factory))];
super::create_plugins(&factories, None, empty_info).expect("one factory is registered")
}
pub(crate) fn invocation_plugins() -> Arc<super::InvocationPlugins> {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
vec![("test::Hooks", Arc::new(Hooks::default()))];
super::create_plugins(&factories, None, empty_info).expect("one factory is registered")
}
pub(crate) fn deadline_in(millis: u64) -> u64 {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap();
u64::try_from(now.as_millis()).unwrap() + millis
}
pub(crate) fn service<F, Fut>(
plugins: Vec<Hooks>,
deadline: u64,
handler: F,
) -> (
impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> crate::BoxedInvocationFuture,
lambda_runtime::LambdaEvent<InvocationPayload>,
)
where
F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<serde_json::Value, BoxError>> + Send,
{
service_with(plugins, deadline, Vec::new(), handler)
}
pub(crate) fn service_with<F, Fut>(
plugins: Vec<Hooks>,
deadline: u64,
recorded: Vec<serde_json::Value>,
handler: F,
) -> (
impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> crate::BoxedInvocationFuture,
lambda_runtime::LambdaEvent<InvocationPayload>,
)
where
F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<serde_json::Value, BoxError>> + Send,
{
service_configured(
move |mut builder| {
for plugin in plugins {
builder = builder.plugin(plugin);
}
builder
},
deadline,
recorded,
handler,
)
}
pub(crate) fn service_configured<F, Fut>(
configure: impl FnOnce(crate::options::OptionsBuilder) -> crate::options::OptionsBuilder,
deadline: u64,
recorded: Vec<serde_json::Value>,
handler: F,
) -> (
impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> crate::BoxedInvocationFuture,
lambda_runtime::LambdaEvent<InvocationPayload>,
)
where
F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<serde_json::Value, BoxError>> + Send,
{
let builder = configure(Options::builder().execution_client(Arc::new(
crate::client::InMemoryExecutionClient::new(Vec::new()),
)));
let mut operations = vec![serde_json::json!({
"Id": "exec-1",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
})];
operations.extend(recorded);
let payload = serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": "tok",
"InitialExecutionState": { "Operations": operations }
});
let mut lambda_ctx = lambda_runtime::Context::default();
lambda_ctx.deadline = deadline;
let event = lambda_runtime::LambdaEvent::new(InvocationPayload(payload), lambda_ctx);
(crate::wrap(handler, builder.build().unwrap()), event)
}
pub(crate) fn capture() -> (Arc<Mutex<Vec<u8>>>, tracing::subscriber::DefaultGuard) {
use tracing_subscriber::layer::SubscriberExt as _;
#[derive(Clone)]
struct Writer(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for Writer {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = Writer(Arc::clone(&buffer));
let layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_writer(move || writer.clone());
let guard =
test_tracing::install_test_subscriber(tracing_subscriber::registry().with(layer));
(buffer, guard)
}
pub(crate) fn warnings(buffer: &Arc<Mutex<Vec<u8>>>) -> Vec<serde_json::Value> {
String::from_utf8_lossy(&buffer.lock().unwrap())
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.filter(|v| {
v.get("target").and_then(serde_json::Value::as_str) == Some(TARGET)
&& v.get("level").and_then(serde_json::Value::as_str) == Some("WARN")
})
.collect()
}
pub(crate) fn field<'a>(event: &'a serde_json::Value, name: &str) -> &'a str {
event
.get(name)
.and_then(serde_json::Value::as_str)
.unwrap_or_default()
}
pub(crate) fn with_message<'a>(
warnings: &'a [serde_json::Value],
message: &str,
) -> Vec<&'a serde_json::Value> {
warnings
.iter()
.filter(|w| field(w, "message") == message)
.collect()
}
pub(crate) struct SignalWaker {
woken: std::sync::atomic::AtomicUsize,
signal: Mutex<std::sync::mpsc::Sender<()>>,
}
impl SignalWaker {
pub(crate) fn new() -> (Arc<Self>, std::sync::mpsc::Receiver<()>) {
let (signal, signals) = std::sync::mpsc::channel();
let waker = Arc::new(Self {
woken: std::sync::atomic::AtomicUsize::new(0),
signal: Mutex::new(signal),
});
(waker, signals)
}
pub(crate) fn woken(&self) -> usize {
self.woken.load(std::sync::atomic::Ordering::SeqCst)
}
}
impl std::task::Wake for SignalWaker {
fn wake(self: Arc<Self>) {
self.wake_by_ref();
}
fn wake_by_ref(self: &Arc<Self>) {
self.woken.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let _receiver_gone = self.signal.lock().unwrap().send(());
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{Duration, UNIX_EPOCH};
fn record(op_type: &str, status: CheckpointStatus) -> CheckpointRecord {
let mut record = CheckpointRecord::with_status("id", status);
record.op_type = Some(op_type.to_owned());
record
}
fn bare_info() -> OperationInfo {
OperationInfo::from_record("a", &record("STEP", CheckpointStatus::Started))
}
#[test]
#[expect(clippy::panic)] fn name_panic_is_logged() {
struct BadName(Arc<AtomicUsize>);
impl Plugin for BadName {
#[expect(clippy::panic, reason = "the test plugin's name panics")]
fn name(&self) -> &'static str {
panic!("name panicked")
}
fn on_user_function_start(&self, _info: &OperationInfo) -> HookWork {
self.0.fetch_add(1, Ordering::SeqCst);
Ok(None)
}
}
struct Factory(Arc<AtomicUsize>);
impl PluginFactory for Factory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(BadName(Arc::clone(&self.0)))))
}
}
for _attempt in 0..5 {
let (buffer, guard) = capture();
let calls = Arc::new(AtomicUsize::new(0));
let factory: Arc<dyn PluginFactory> = Arc::new(Factory(Arc::clone(&calls)));
let plugins = create_plugins(&[("Factory", factory)], None, tests_support_info)
.expect("factories were given");
assert_eq!(
plugins.plugins.first().map(|s| s.identity.as_str()),
Some("plugins[0]")
);
let started = plugins.dispatch_to("on_user_function_start", [0], |plugin| {
plugin.on_user_function_start(&OperationInfo::for_test("s"))
});
assert_eq!(started, [0], "the start hook still runs and succeeds");
assert_eq!(calls.load(Ordering::SeqCst), 1);
drop(guard);
let warnings = warnings(&buffer);
let [warning] = warnings.as_slice() else {
continue;
};
assert_eq!(field(warning, "plugin"), "plugins[0]");
assert_eq!(field(warning, "hook"), "name");
assert_eq!(
field(warning, "message"),
"plugin hook panicked; the panic is ignored"
);
return;
}
panic!("no attempt captured exactly one name() warning");
}
#[test]
fn a_hook_errors_display_calling_an_operation_fails_the_execution() {
#[derive(Debug)]
struct Calls(DurableContext);
impl fmt::Display for Calls {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
drop(self.0.claim_operation_id("wait"));
f.write_str("calls")
}
}
impl std::error::Error for Calls {}
struct Failing(DurableContext);
impl Plugin for Failing {
fn on_operation_start(&self, _info: &OperationInfo) -> HookWork {
Err(Box::new(Calls(self.0.clone())))
}
}
struct Factory(DurableContext);
impl PluginFactory for Factory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(Failing(self.0.clone()))))
}
}
let ctx = DurableContext::test_context();
let factory: Arc<dyn PluginFactory> = Arc::new(Factory(ctx.clone()));
let plugins = create_plugins(&[("Factory", factory)], None, tests_support_info)
.expect("factories were given");
plugins.dispatch_work("on_operation_start", |plugin| {
plugin.on_operation_start(&OperationInfo::for_test("s"))
});
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert!(
fatal.error_message.contains("inside plugin code"),
"{}",
fatal.error_message
);
}
#[test]
fn a_panic_payloads_destructor_calling_an_operation_fails_the_execution() {
struct Calls(DurableContext);
impl Drop for Calls {
fn drop(&mut self) {
drop(self.0.claim_operation_id("wait"));
}
}
struct Panicking(DurableContext);
impl Plugin for Panicking {
#[expect(clippy::panic, reason = "the test hook panics with a payload")]
fn on_operation_start(&self, _info: &OperationInfo) -> HookWork {
std::panic::panic_any(Calls(self.0.clone()))
}
}
struct Factory(DurableContext);
impl PluginFactory for Factory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(Panicking(self.0.clone()))))
}
}
let ctx = DurableContext::test_context();
let factory: Arc<dyn PluginFactory> = Arc::new(Factory(ctx.clone()));
let plugins = create_plugins(&[("Factory", factory)], None, tests_support_info)
.expect("factories were given");
plugins.dispatch_work("on_operation_start", |plugin| {
plugin.on_operation_start(&OperationInfo::for_test("s"))
});
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert!(
fatal.error_message.contains("inside plugin code"),
"{}",
fatal.error_message
);
}
#[test]
fn a_rejected_kept_payloads_destructor_calling_an_operation_fails_the_execution() {
struct Calls(DurableContext);
impl Drop for Calls {
fn drop(&mut self) {
drop(self.0.claim_operation_id("wait"));
}
}
#[derive(Debug)]
struct DropPanics(Option<DurableContext>);
impl fmt::Display for DropPanics {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("factory failed")
}
}
impl std::error::Error for DropPanics {}
impl Drop for DropPanics {
#[expect(clippy::panic, reason = "the destructor panic under test")]
fn drop(&mut self) {
match self.0.take() {
Some(ctx) => std::panic::panic_any(Calls(ctx)),
None => panic!("first error drop"),
}
}
}
struct Factory(Option<DurableContext>);
impl PluginFactory for Factory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Err(Box::new(DropPanics(self.0.clone())))
}
}
let ctx = DurableContext::test_context();
let first: Arc<dyn PluginFactory> = Arc::new(Factory(None));
let second: Arc<dyn PluginFactory> = Arc::new(Factory(Some(ctx.clone())));
let plugins = create_plugins(
&[("First", first), ("Second", second)],
None,
tests_support_info,
)
.expect("factories were given");
assert!(ctx.coordinator().fatal_error().is_none());
plugins.attach(ctx.coordinator());
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert!(
fatal.error_message.contains("inside plugin code"),
"{}",
fatal.error_message
);
}
#[tokio::test]
#[expect(clippy::panic)] async fn subscriber_in_plugin_warning_calls_durable_op() {
type Slot = Arc<Mutex<Option<DurableContext>>>;
struct Calling(Slot);
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Calling {
fn on_event(
&self,
event: &tracing::Event<'_>,
_ctx: tracing_subscriber::layer::Context<'_, S>,
) {
if event.metadata().target() == TARGET
&& let Some(ctx) = self.0.lock().unwrap().clone()
{
drop(ctx.step(|_| async { Ok::<u32, BoxError>(1) }));
}
}
}
struct FailingStart;
impl Plugin for FailingStart {
fn on_user_function_start(&self, _info: &OperationInfo) -> HookWork {
Err("start failed".into())
}
}
impl PluginFactory for FailingStart {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(FailingStart)))
}
}
use tracing_subscriber::layer::SubscriberExt as _;
for _attempt in 0..5 {
let slot: Slot = Arc::default();
let _guard = test_tracing::install_test_subscriber(
tracing_subscriber::registry().with(Calling(Arc::clone(&slot))),
);
let (service, event) = test_support::service_configured(
|builder| builder.plugin(FailingStart),
0,
Vec::new(),
move |_e, ctx| {
*slot.lock().unwrap() = Some(ctx.clone());
async move {
let value = ctx.step(|_| async { Ok(7_u32) }).name("s").await?;
Ok(serde_json::json!(value))
}
},
);
let json = service(event).await.unwrap().0;
if json.get("Status") == Some(&serde_json::json!("SUCCEEDED")) {
continue;
}
assert_eq!(json.get("Status"), Some(&serde_json::json!("FAILED")));
assert!(json.to_string().contains("inside plugin code"), "{json}");
return;
}
panic!("no attempt delivered the warning to the subscriber");
}
#[test]
fn for_operation_without_record_holds_the_identity() {
let ctx = DurableContext::test_context();
let info = OperationInfo::for_operation(&ctx, "w", OperationKind::Step, "Pay", Some("a"))
.with_status(OperationStatus::Started);
assert_eq!(info.id(), "w");
assert_eq!(info.kind(), Some(OperationKind::Step));
assert_eq!(info.type_str(), Some("STEP"));
assert_eq!(info.sub_type(), Some("Pay"));
assert_eq!(info.name(), Some("a"));
assert_eq!(info.parent_id(), None);
assert_eq!(info.status(), Some(OperationStatus::Started));
assert_eq!(info.status_str(), Some("STARTED"));
assert_eq!(info.start_timestamp(), None);
assert_eq!(info.attempt(), None);
}
#[test]
fn with_virtual_sets_the_flag() {
let ctx = DurableContext::test_context();
let info = OperationInfo::for_operation(&ctx, "w", OperationKind::Context, "Map", None);
assert!(!info.is_virtual());
let info = info.with_virtual(true);
assert!(info.is_virtual());
assert!(!info.with_virtual(false).is_virtual());
}
#[test]
fn for_operation_without_record_takes_the_context_parent() {
let ctx = DurableContext::test_context().new_child("1", None);
let parent = ctx
.parent_wire_id()
.expect("a child has a parent")
.to_owned();
let info = OperationInfo::for_operation(&ctx, "w", OperationKind::Step, "Step", None);
assert_eq!(info.parent_id(), Some(parent.as_str()));
assert_eq!(info.name(), None);
}
#[test]
fn for_operation_keeps_populated_record_fields() {
let mut rec = record("STEP", CheckpointStatus::Started);
rec.op_name = Some("recorded".to_owned());
rec.sub_type = Some("Recorded".to_owned());
rec.parent_id = Some("p".to_owned());
let ctx = DurableContext::new_root(
String::from("arn"),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(vec![("w".to_owned(), rec)])),
);
let info = OperationInfo::for_operation(
&ctx.new_child("1", None),
"w",
OperationKind::Wait,
"Pay",
Some("a"),
)
.with_status(OperationStatus::Succeeded);
assert_eq!(info.kind(), Some(OperationKind::Step));
assert_eq!(info.type_str(), Some("STEP"));
assert_eq!(info.sub_type(), Some("Recorded"));
assert_eq!(info.name(), Some("recorded"));
assert_eq!(info.parent_id(), Some("p"));
assert_eq!(info.status(), Some(OperationStatus::Succeeded));
assert_eq!(info.status_str(), Some("SUCCEEDED"));
}
#[test]
fn for_operation_fills_absent_record_fields() {
let rec = CheckpointRecord::with_status("w", CheckpointStatus::Started);
let ctx = DurableContext::new_root(
String::from("arn"),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(vec![("w".to_owned(), rec)])),
);
let info = OperationInfo::for_operation(&ctx, "w", OperationKind::Step, "Pay", Some("a"));
assert_eq!(info.kind(), Some(OperationKind::Step));
assert_eq!(info.type_str(), Some("STEP"));
assert_eq!(info.sub_type(), Some("Pay"));
assert_eq!(info.name(), Some("a"));
assert_eq!(info.parent_id(), None);
assert_eq!(info.status_str(), Some("STARTED"));
}
#[test]
fn observed_end_keeps_the_recorded_timestamp() {
let recorded = UNIX_EPOCH + Duration::from_secs(5);
let info = with_observed_end(bare_info().with_end_timestamp(recorded));
assert_eq!(info.end_timestamp(), Some(recorded));
}
#[test]
fn observed_end_uses_local_time_without_record() {
let info = bare_info();
assert_eq!(info.end_timestamp(), None);
let before = SystemTime::now();
let info = with_observed_end(info);
let after = SystemTime::now();
let end = info.end_timestamp().expect("an end timestamp");
assert!(before <= end && end <= after, "{end:?}");
}
#[test]
fn operation_info_from_record_maps_every_field() {
let mut rec = record("STEP", CheckpointStatus::Succeeded);
rec.sub_type = Some("Step".to_owned());
rec.op_name = Some("n".to_owned());
rec.parent_id = Some("p".to_owned());
rec.result = Some("\"x\"".to_owned());
rec.attempt = 2;
rec.start_timestamp = Some(UNIX_EPOCH + Duration::from_secs(1));
rec.end_timestamp = Some(UNIX_EPOCH + Duration::from_secs(2));
let info = OperationInfo::from_record("op-1", &rec);
assert_eq!(info.id(), "op-1");
assert_eq!(info.name(), Some("n"));
assert_eq!(info.kind(), Some(OperationKind::Step));
assert_eq!(info.type_str(), Some("STEP"));
assert_eq!(info.sub_type(), Some("Step"));
assert_eq!(info.parent_id(), Some("p"));
assert_eq!(info.status(), Some(OperationStatus::Succeeded));
assert_eq!(info.status_str(), Some("SUCCEEDED"));
assert_eq!(
info.start_timestamp(),
Some(UNIX_EPOCH + Duration::from_secs(1))
);
assert_eq!(
info.end_timestamp(),
Some(UNIX_EPOCH + Duration::from_secs(2))
);
assert_eq!(info.result(), Some("\"x\""));
assert!(info.error().is_none());
assert_eq!(info.attempt(), Some(2));
assert!(!info.is_replay());
assert!(!info.is_virtual());
}
#[test]
fn operation_info_reads_both_type_spellings() {
for spelling in ["ChainedInvoke", "CHAINED_INVOKE"] {
let info =
OperationInfo::from_record("a", &record(spelling, CheckpointStatus::Started));
assert_eq!(
info.kind(),
Some(OperationKind::ChainedInvoke),
"{spelling}"
);
assert_eq!(info.type_str(), Some("CHAINED_INVOKE"), "{spelling}");
}
}
#[test]
fn operation_info_unknown_type_and_status_are_none() {
let info = OperationInfo::from_record(
"a",
&record("BOGUS", CheckpointStatus::Unknown("NEW".to_owned())),
);
assert_eq!(info.kind(), None);
assert_eq!(info.status(), None);
assert_eq!(info.type_str(), Some("BOGUS"));
assert_eq!(info.status_str(), Some("NEW"));
}
#[test]
fn operation_info_without_type_has_no_kind() {
let rec = CheckpointRecord::with_status("id", CheckpointStatus::Started);
let info = OperationInfo::from_record("a", &rec);
assert_eq!(info.kind(), None);
assert_eq!(info.type_str(), None);
}
#[test]
fn operation_info_maps_every_known_status() {
for (stored, typed) in [
(CheckpointStatus::Started, OperationStatus::Started),
(CheckpointStatus::Pending, OperationStatus::Pending),
(CheckpointStatus::Ready, OperationStatus::Ready),
(CheckpointStatus::Succeeded, OperationStatus::Succeeded),
(CheckpointStatus::Failed, OperationStatus::Failed),
(CheckpointStatus::Cancelled, OperationStatus::Cancelled),
(CheckpointStatus::TimedOut, OperationStatus::TimedOut),
(CheckpointStatus::Stopped, OperationStatus::Stopped),
] {
let info = OperationInfo::from_record("a", &record("STEP", stored));
assert_eq!(info.status(), Some(typed));
assert_eq!(info.status_str(), Some(typed.wire_str()));
}
}
#[test]
fn operation_info_error_absent_when_no_error_field() {
let info = OperationInfo::from_record("a", &record("STEP", CheckpointStatus::Failed));
assert!(info.error().is_none());
assert_eq!(info.status(), Some(OperationStatus::Failed));
}
#[test]
fn operation_info_error_kept_with_message_only() {
let mut rec = record("STEP", CheckpointStatus::Failed);
rec.error_message = Some(String::new());
let info = OperationInfo::from_record("a", &rec);
let error = info.error().expect("an error with an empty message");
assert_eq!(error.error_message(), Some(""));
assert_eq!(error.error_type(), None);
}
#[test]
fn operation_info_error_kept_with_any_field() {
let mut data_only = record("STEP", CheckpointStatus::Failed);
data_only.error_data = Some("d".to_owned());
let info = OperationInfo::from_record("a", &data_only);
assert_eq!(info.error().and_then(WireError::error_data), Some("d"));
let mut trace_only = record("STEP", CheckpointStatus::Failed);
trace_only.stack_trace = Some(vec!["frame".to_owned()]);
let info = OperationInfo::from_record("a", &trace_only);
assert_eq!(
info.error().map(WireError::stack_trace),
Some(&["frame".to_owned()][..])
);
let mut type_only = record("STEP", CheckpointStatus::Failed);
type_only.error_type = Some("T".to_owned());
let info = OperationInfo::from_record("a", &type_only);
assert_eq!(info.error().and_then(WireError::error_type), Some("T"));
}
#[test]
fn operation_info_reads_invoke_payloads() {
let mut rec = record("CHAINED_INVOKE", CheckpointStatus::Succeeded);
rec.invoke_result = Some("\"r\"".to_owned());
rec.invoke_error_type = Some("E".to_owned());
let info = OperationInfo::from_record("a", &rec);
assert_eq!(info.result(), Some("\"r\""));
assert_eq!(info.error().and_then(WireError::error_type), Some("E"));
}
#[test]
fn operation_info_prefers_record_payloads_over_invoke() {
let mut rec = record("STEP", CheckpointStatus::Failed);
rec.result = Some("own".to_owned());
rec.invoke_result = Some("invoke".to_owned());
rec.error_type = Some("Own".to_owned());
rec.invoke_error_type = Some("Invoke".to_owned());
let info = OperationInfo::from_record("a", &rec);
assert_eq!(info.result(), Some("own"));
assert_eq!(info.error().and_then(WireError::error_type), Some("Own"));
let mut message = record("CHAINED_INVOKE", CheckpointStatus::Failed);
message.invoke_error_message = Some("m".to_owned());
message.invoke_error_data = Some("d".to_owned());
message.invoke_stack_trace = Some(vec!["f".to_owned()]);
let error = OperationInfo::from_record("a", &message)
.error()
.cloned()
.expect("invoke error");
assert_eq!(error.error_message(), Some("m"));
assert_eq!(error.error_data(), Some("d"));
assert_eq!(error.stack_trace(), ["f".to_owned()]);
}
#[test]
fn operation_info_attempt_zero_is_none() {
let info = OperationInfo::from_record("a", &record("STEP", CheckpointStatus::Started));
assert_eq!(info.attempt(), None);
}
#[test]
fn execution_info_requires_execution_type() {
assert!(
OperationInfo::execution("s", "STEP", &CheckpointStatus::Started, None, None, None)
.is_none()
);
let start = UNIX_EPOCH + Duration::from_secs(3);
let info = OperationInfo::execution(
"e",
"EXECUTION",
&CheckpointStatus::Started,
Some("p".to_owned()),
Some(start),
None,
)
.expect("an EXECUTION operation");
assert_eq!(info.id(), "e");
assert_eq!(info.kind(), Some(OperationKind::Execution));
assert_eq!(info.type_str(), Some("EXECUTION"));
assert_eq!(info.status(), Some(OperationStatus::Started));
assert_eq!(info.status_str(), Some("STARTED"));
assert_eq!(info.parent_id(), Some("p"));
assert_eq!(info.start_timestamp(), Some(start));
assert_eq!(info.end_timestamp(), None);
assert_eq!(info.name(), None);
assert_eq!(info.result(), None);
assert_eq!(info.attempt(), None);
}
fn info_with(ops: &[&str], updated: &[&str]) -> InvocationInfo {
let operations = ops
.iter()
.map(|id| {
(
(*id).to_owned(),
OperationInfo::from_record(id, &record("STEP", CheckpointStatus::Started)),
)
})
.collect();
InvocationInfo::new(
"r".to_owned(),
"arn".to_owned(),
"{}".to_owned(),
operations,
updated.iter().map(|id| (*id).to_owned()).collect(),
None,
false,
None,
)
}
#[test]
fn invocation_info_filters_updated_ids() {
let info = info_with(&["a", "b"], &["b", "x", "b", "a"]);
let ids: Vec<&str> = info
.updated_operations()
.iter()
.map(OperationInfo::id)
.collect();
assert_eq!(ids, ["b", "a"]);
}
#[test]
fn invocation_info_accessors_return_fields() {
let start = UNIX_EPOCH + Duration::from_secs(5);
let info = InvocationInfo::new(
"req".to_owned(),
"arn:x".to_owned(),
"\"in\"".to_owned(),
BTreeMap::new(),
Vec::new(),
Some(start),
true,
Some("Root=1".to_owned()),
);
assert_eq!(info.request_id(), "req");
assert_eq!(info.execution_arn(), "arn:x");
assert_eq!(info.execution_input(), "\"in\"");
assert!(info.operations().is_empty());
assert!(info.updated_operations().is_empty());
assert_eq!(info.execution_start_timestamp(), Some(start));
assert!(info.is_first_invocation());
assert_eq!(info.x_ray_trace_id(), Some("Root=1"));
}
enum Behavior {
Some,
None,
Panic,
Err,
PanickingName,
PanickingError,
ErrorPanickingDrop,
ErrorPanickingDisplayAndDrop,
PanickingDrop,
ErrorPanickingNestedDrop,
}
struct TestFactory {
label: &'static str,
behavior: Behavior,
log: Arc<Mutex<Vec<String>>>,
}
struct Dropping {
label: &'static str,
log: Arc<Mutex<Vec<String>>>,
}
impl Plugin for Dropping {}
impl Drop for Dropping {
fn drop(&mut self) {
self.log
.lock()
.unwrap()
.push(format!("dropped {}", self.label));
}
}
struct PanickingName;
impl Plugin for PanickingName {
#[expect(clippy::panic)] fn name(&self) -> &'static str {
panic!("name boom")
}
}
#[derive(Debug)]
struct PanickingDisplay;
impl fmt::Display for PanickingDisplay {
#[expect(clippy::panic)] fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
panic!("display boom")
}
}
impl std::error::Error for PanickingDisplay {}
#[derive(Debug)]
struct PanickingDropError {
display_panics: bool,
}
impl fmt::Display for PanickingDropError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
assert!(!self.display_panics, "display boom");
f.write_str("drop err")
}
}
impl std::error::Error for PanickingDropError {}
impl Drop for PanickingDropError {
#[expect(clippy::panic)] fn drop(&mut self) {
panic!("error drop");
}
}
struct PanickingPayload;
impl Drop for PanickingPayload {
#[expect(clippy::panic)] fn drop(&mut self) {
panic!("payload drop");
}
}
#[derive(Debug)]
struct NestedDropError;
impl fmt::Display for NestedDropError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("nested")
}
}
impl std::error::Error for NestedDropError {}
impl Drop for NestedDropError {
#[expect(clippy::panic)] fn drop(&mut self) {
std::panic::panic_any(PanickingPayload);
}
}
struct PanickingDropPlugin;
impl Plugin for PanickingDropPlugin {}
impl Drop for PanickingDropPlugin {
#[expect(clippy::panic)] fn drop(&mut self) {
panic!("plugin drop");
}
}
impl PluginFactory for TestFactory {
#[expect(clippy::panic)] fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
self.log.lock().unwrap().push(self.label.to_owned());
match self.behavior {
Behavior::Some => Ok(Some(Box::new(Dropping {
label: self.label,
log: Arc::clone(&self.log),
}))),
Behavior::None => Ok(None),
Behavior::Panic => panic!("factory boom"),
Behavior::Err => Err("x".into()),
Behavior::PanickingName => Ok(Some(Box::new(PanickingName))),
Behavior::PanickingError => Err(Box::new(PanickingDisplay)),
Behavior::ErrorPanickingDrop => Err(Box::new(PanickingDropError {
display_panics: false,
})),
Behavior::ErrorPanickingDisplayAndDrop => Err(Box::new(PanickingDropError {
display_panics: true,
})),
Behavior::PanickingDrop => Ok(Some(Box::new(PanickingDropPlugin))),
Behavior::ErrorPanickingNestedDrop => Err(Box::new(NestedDropError)),
}
}
}
fn factories(
specs: Vec<(&'static str, Behavior)>,
log: &Arc<Mutex<Vec<String>>>,
) -> Vec<(&'static str, Arc<dyn PluginFactory>)> {
specs
.into_iter()
.map(|(label, behavior)| {
(
"test::TestFactory",
Arc::new(TestFactory {
label,
behavior,
log: Arc::clone(log),
}) as Arc<dyn PluginFactory>,
)
})
.collect()
}
fn empty_info() -> InvocationInfo {
info_with(&[], &[])
}
#[test]
#[expect(clippy::panic)] fn create_plugins_without_factories_builds_no_info() {
let result = create_plugins(&[], None, || panic!("build_info must not run"));
assert!(result.is_none());
}
use super::test_support::{capture, field, warnings};
#[test]
#[expect(clippy::panic)] fn create_plugins_skips_none_and_panicking_factories() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(
vec![
("A", Behavior::Some),
("none", Behavior::None),
("panic", Behavior::Panic),
("B", Behavior::Some),
("err", Behavior::Err),
],
&log,
);
let plugins = create_plugins(&factories, None, empty_info);
assert!(plugins.is_some());
drop(plugins);
assert_eq!(
*log.lock().unwrap(),
["A", "none", "panic", "B", "err", "dropped A", "dropped B"]
);
let warnings = warnings(&buffer);
if warnings.len() != 2 {
continue;
}
let panic = warnings.first().unwrap();
assert!(field(panic, "factory").ends_with("(plugins[2])"), "{panic}");
assert!(field(panic, "message").contains("panicked"), "{panic}");
let err = warnings.get(1).unwrap();
assert!(field(err, "factory").ends_with("(plugins[4])"), "{err}");
assert_eq!(field(err, "error"), "x");
return;
}
panic!("no attempt captured the two factory warnings");
}
#[test]
fn create_plugins_returns_some_when_every_factory_skips() {
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(vec![("none", Behavior::None)], &log);
assert!(create_plugins(&factories, None, empty_info).is_some());
}
#[test]
#[expect(clippy::panic)] fn create_plugins_contains_a_panicking_error_display() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(
vec![("bad", Behavior::PanickingError), ("A", Behavior::Some)],
&log,
);
let mut error_drop_panics = Vec::new();
let slots = create_slots(&factories, &empty_info(), &mut error_drop_panics);
assert_eq!(slots.len(), 1);
assert!(error_drop_panics.is_empty());
let warnings = warnings(&buffer);
let [warning] = warnings.as_slice() else {
continue;
};
assert!(
field(warning, "factory").ends_with("(plugins[0])"),
"{warning}"
);
assert!(
field(warning, "message").contains("panicked while it was formatted"),
"{warning}"
);
return;
}
panic!("no attempt captured the warning of the panicking error");
}
#[test]
#[expect(clippy::panic)] fn factory_error_destructor_panic_is_kept_for_the_coordinator() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(
vec![("bad", Behavior::ErrorPanickingDrop), ("A", Behavior::Some)],
&log,
);
let plugins = create_plugins(&factories, None, empty_info).unwrap();
let panics = plugins.take_error_drop_panics();
let [payload] = panics.as_slice() else {
panic!("one destructor panic is kept, got {}", panics.len());
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
assert!(plugins.take_error_drop_panics().is_empty());
drop(plugins);
assert_eq!(*log.lock().unwrap(), ["bad", "A", "dropped A"]);
let warnings = warnings(&buffer);
let [warning] = warnings.as_slice() else {
continue;
};
assert!(
field(warning, "factory").ends_with("(plugins[0])"),
"{warning}"
);
assert_eq!(field(warning, "error"), "drop err");
return;
}
panic!("no attempt captured the warning of the failed factory");
}
#[test]
#[expect(clippy::panic)] fn factory_error_display_and_destructor_panics_do_not_abort() {
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(vec![("bad", Behavior::ErrorPanickingDisplayAndDrop)], &log);
let plugins = create_plugins(&factories, None, empty_info).unwrap();
let panics = plugins.take_error_drop_panics();
let [payload] = panics.as_slice() else {
panic!("one destructor panic is kept, got {}", panics.len());
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
}
#[test]
fn untaken_error_drop_panic_does_not_escape() {
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(vec![("bad", Behavior::ErrorPanickingNestedDrop)], &log);
let plugins = create_plugins(&factories, None, empty_info).unwrap();
let result = catch_unwind(AssertUnwindSafe(move || drop(plugins)));
assert!(result.is_ok(), "no panic escapes the drop");
}
#[test]
fn two_plugin_drop_panics_do_not_abort() {
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(
vec![
("bad", Behavior::PanickingDrop),
("bad2", Behavior::PanickingDrop),
("B", Behavior::Some),
],
&log,
);
let plugins = create_plugins(&factories, None, empty_info);
let result = catch_unwind(AssertUnwindSafe(move || drop(plugins)));
assert!(result.is_ok(), "no panic escapes the drop");
assert_eq!(*log.lock().unwrap(), ["bad", "bad2", "B", "dropped B"]);
}
#[test]
#[expect(clippy::panic)] fn plugin_drop_panic_is_logged_and_discarded() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(
vec![("bad", Behavior::PanickingDrop), ("B", Behavior::Some)],
&log,
);
let plugins = create_plugins(&factories, None, empty_info);
drop(plugins);
assert_eq!(*log.lock().unwrap(), ["bad", "B", "dropped B"]);
let warnings = warnings(&buffer);
let [warning] = warnings.as_slice() else {
continue;
};
assert!(
field(warning, "plugin").ends_with("PanickingDropPlugin (plugins[0])"),
"{warning}"
);
assert_eq!(field(warning, "hook"), "drop");
return;
}
panic!("no attempt captured the warning of the panicking plugin destructor");
}
#[test]
fn plugin_drop_panic_during_unwind_does_not_abort() {
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(
vec![("bad", Behavior::PanickingDrop), ("B", Behavior::Some)],
&log,
);
let plugins = create_plugins(&factories, None, empty_info);
let result = catch_unwind(AssertUnwindSafe(move || {
let _plugins = plugins;
std::panic::resume_unwind(Box::new("outer"));
}));
let payload = result.expect_err("the outer panic unwinds");
assert_eq!(payload.downcast_ref::<&str>(), Some(&"outer"));
assert_eq!(*log.lock().unwrap(), ["bad", "B", "dropped B"]);
}
struct Probe;
impl Plugin for Probe {}
#[test]
fn plugin_name_defaults_to_type_path() {
assert!(Probe.name().ends_with("Probe"), "{}", Probe.name());
}
struct ProbeFactory(bool);
impl PluginFactory for ProbeFactory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(self.0.then(|| Box::new(Probe) as Box<dyn Plugin>))
}
}
#[test]
fn identity_uses_registration_index() {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> = vec![
("skip", Arc::new(ProbeFactory(false))),
("probe", Arc::new(ProbeFactory(true))),
];
let slots = create_slots(&factories, &empty_info(), &mut Vec::new());
assert_eq!(slots.len(), 1);
let slot = slots.first().unwrap();
assert!(
slot.identity.ends_with("Probe (plugins[1])"),
"{}",
slot.identity
);
assert!(slot.plugin.name().ends_with("Probe"));
}
#[test]
fn identity_falls_back_when_name_panics() {
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(vec![("n", Behavior::PanickingName)], &log);
let slots = create_slots(&factories, &empty_info(), &mut Vec::new());
assert_eq!(slots.len(), 1);
assert_eq!(slots.first().unwrap().identity, "plugins[0]");
}
fn step_records(count: usize) -> Vec<(String, CheckpointRecord)> {
(0..count)
.map(|i| {
let id = format!("op-{i}");
(id, record("STEP", CheckpointStatus::Succeeded))
})
.collect()
}
#[test]
fn operation_infos_projects_large_log_once() {
let log = CheckpointLog::from_records(step_records(10_000));
let before = crate::engine::projections();
let map = operation_infos(&log);
assert_eq!(map.len(), 10_000);
assert_eq!(crate::engine::projections() - before, 1);
}
fn change_entry(id: &str) -> Vec<OperationInfo> {
vec![OperationInfo::from_record(
id,
&record("STEP", CheckpointStatus::Started),
)]
}
#[test]
fn drain_delivers_in_order_and_empties_queue() {
let recorder = test_support::Changes::default();
let plugins = test_support::plugins_of(recorder.clone());
let log = CheckpointLog::empty();
for id in ["a", "b", "c"] {
plugins.push_operation_change(change_entry(id));
}
plugins.drain_operation_changes(&log);
assert_eq!(recorder.updated_ids(), [["a"], ["b"], ["c"]]);
plugins.drain_operation_changes(&log);
assert_eq!(recorder.calls().len(), 3, "a second drain makes no call");
}
#[test]
fn drain_is_skipped_while_another_drain_runs() {
let target: Arc<std::sync::OnceLock<Weak<InvocationPlugins>>> = Arc::default();
let inner_calls = Arc::new(Mutex::new(None::<usize>));
let mut recorder = test_support::Changes::default();
let first = Arc::new(AtomicBool::new(true));
{
let target = Arc::clone(&target);
let calls = Arc::clone(&recorder.calls);
let inner_calls = Arc::clone(&inner_calls);
recorder.on_change = Some(Arc::new(move |_info| {
if !first.swap(false, Ordering::SeqCst) {
return;
}
let plugins = target.get().unwrap().upgrade().unwrap();
plugins.push_operation_change(change_entry("d"));
plugins.drain_operation_changes(&CheckpointLog::empty());
*inner_calls.lock().unwrap() = Some(calls.lock().unwrap().len());
}));
}
let plugins = test_support::plugins_of(recorder.clone());
target.set(Arc::downgrade(&plugins)).unwrap();
for id in ["a", "b", "c"] {
plugins.push_operation_change(change_entry(id));
}
plugins.drain_operation_changes(&CheckpointLog::empty());
assert_eq!(
*inner_calls.lock().unwrap(),
Some(1),
"the inner drain made no call"
);
assert_eq!(recorder.updated_ids(), [["a"], ["b"], ["c"], ["d"]]);
assert!(!plugins.is_draining());
}
#[test]
fn drain_rechecks_queue_after_clearing_flag() {
let recorder = test_support::Changes::default();
let plugins = test_support::plugins_of(recorder.clone());
let log = CheckpointLog::empty();
plugins.set_after_release(|plugins| {
assert!(
!plugins.is_draining(),
"the hook runs after the flag is clear"
);
plugins.push_operation_change(change_entry("b"));
});
plugins.push_operation_change(change_entry("a"));
plugins.drain_operation_changes(&log);
assert_eq!(recorder.updated_ids(), [["a"], ["b"]]);
assert!(!plugins.is_draining());
plugins.drain_operation_changes(&log);
assert_eq!(recorder.calls().len(), 2, "the entry was delivered once");
}
fn noop_context() -> Context<'static> {
Context::from_waker(std::task::Waker::noop())
}
struct HeldDrain {
plugins: Arc<InvocationPlugins>,
log: Arc<CheckpointLog>,
recorder: test_support::Changes,
release: std::sync::mpsc::Sender<()>,
holder: std::thread::JoinHandle<()>,
}
impl HeldDrain {
fn start() -> Self {
let (entered_tx, entered_rx) = std::sync::mpsc::channel::<()>();
let (release, release_rx) = std::sync::mpsc::channel::<()>();
let release_rx = Mutex::new(release_rx);
let first = AtomicBool::new(true);
let recorder = test_support::Changes {
on_change: Some(Arc::new(move |_info| {
if first.swap(false, Ordering::SeqCst) {
entered_tx.send(()).unwrap();
release_rx.lock().unwrap().recv().unwrap();
}
})),
..test_support::Changes::default()
};
let plugins = test_support::plugins_of(recorder.clone());
let log = Arc::new(CheckpointLog::empty());
plugins.push_operation_change(change_entry("a"));
let holder = {
let plugins = Arc::clone(&plugins);
let log = Arc::clone(&log);
std::thread::spawn(move || plugins.drain_operation_changes(&log))
};
entered_rx.recv().unwrap();
plugins.push_operation_change(change_entry("b"));
Self {
plugins,
log,
recorder,
release,
holder,
}
}
fn finish(self) {
self.release.send(()).unwrap();
self.holder.join().unwrap();
}
}
#[test]
fn delivery_wakes_registered_waiter() {
let held = HeldDrain::start();
let (signal, signals) = test_support::SignalWaker::new();
let waker = std::task::Waker::from(Arc::clone(&signal));
let mut cx = Context::from_waker(&waker);
let mut waiter = Box::pin(held.plugins.changes_delivered(&held.log));
assert!(waiter.as_mut().poll(&mut cx).is_pending());
assert_eq!(held.plugins.delivery_waiter_count(), 1);
assert_eq!(signal.woken(), 0);
assert_eq!(held.recorder.updated_ids(), [["a"]]);
held.release.send(()).unwrap();
signals
.recv_timeout(Duration::from_secs(30))
.expect("a delivery woke the registered waker");
held.holder.join().unwrap();
assert!(waiter.as_mut().poll(&mut cx).is_ready());
assert_eq!(held.plugins.delivery_waiter_count(), 0);
assert_eq!(held.recorder.updated_ids(), [["a"], ["b"]]);
}
#[test]
fn repeated_polls_keep_one_registration() {
let held = HeldDrain::start();
let (first, _first_signals) = test_support::SignalWaker::new();
let (latest, latest_signals) = test_support::SignalWaker::new();
let first_waker = std::task::Waker::from(Arc::clone(&first));
let latest_waker = std::task::Waker::from(Arc::clone(&latest));
let mut waiter = Box::pin(held.plugins.changes_delivered(&held.log));
for _ in 0..100 {
assert!(
waiter
.as_mut()
.poll(&mut Context::from_waker(&first_waker))
.is_pending()
);
}
assert_eq!(held.plugins.delivery_waiter_count(), 1);
assert!(
waiter
.as_mut()
.poll(&mut Context::from_waker(&latest_waker))
.is_pending()
);
assert_eq!(held.plugins.delivery_waiter_count(), 1);
held.release.send(()).unwrap();
latest_signals
.recv_timeout(Duration::from_secs(30))
.expect("the delivery woke the latest waker");
held.holder.join().unwrap();
assert_eq!(first.woken(), 0, "the replaced waker is not woken");
assert!(
waiter
.as_mut()
.poll(&mut Context::from_waker(&latest_waker))
.is_ready()
);
assert_eq!(held.plugins.delivery_waiter_count(), 0);
}
#[test]
fn dropped_waiter_removes_registration() {
let held = HeldDrain::start();
let mut waiter = Box::pin(held.plugins.changes_delivered(&held.log));
assert!(waiter.as_mut().poll(&mut noop_context()).is_pending());
assert_eq!(held.plugins.delivery_waiter_count(), 1);
drop(waiter);
assert_eq!(held.plugins.delivery_waiter_count(), 0);
let plugins = Arc::clone(&held.plugins);
let recorder = held.recorder.clone();
held.finish();
assert_eq!(recorder.updated_ids(), [["a"], ["b"]]);
assert_eq!(plugins.delivery_waiter_count(), 0);
}
#[test]
fn waiter_on_delivering_thread_is_ready() {
let target: Arc<std::sync::OnceLock<Weak<InvocationPlugins>>> = Arc::default();
let inner = Arc::new(Mutex::new(None::<bool>));
let mut recorder = test_support::Changes::default();
{
let target = Arc::clone(&target);
let inner = Arc::clone(&inner);
let first = AtomicBool::new(true);
recorder.on_change = Some(Arc::new(move |_info| {
if !first.swap(false, Ordering::SeqCst) {
return;
}
let plugins = target.get().unwrap().upgrade().unwrap();
plugins.push_operation_change(change_entry("b"));
let log = CheckpointLog::empty();
let mut waiter = Box::pin(plugins.changes_delivered(&log));
let ready = waiter.as_mut().poll(&mut noop_context()).is_ready();
inner.lock().unwrap().get_or_insert(ready);
}));
}
let plugins = test_support::plugins_of(recorder.clone());
target.set(Arc::downgrade(&plugins)).unwrap();
plugins.push_operation_change(change_entry("a"));
let log = CheckpointLog::empty();
let mut waiter = Box::pin(plugins.changes_delivered(&log));
assert!(waiter.as_mut().poll(&mut noop_context()).is_ready());
assert_eq!(*inner.lock().unwrap(), Some(true));
assert_eq!(recorder.updated_ids(), [["a"], ["b"]]);
}
#[test]
fn panicking_change_hook_is_isolated() {
let recorder = test_support::Changes::default();
let panics = test_support::Changes {
panics: true,
..test_support::Changes::default()
};
let plugins = test_support::plugins_of_all(vec![
Arc::new(panics) as Arc<dyn PluginFactory>,
Arc::new(recorder.clone()),
]);
let log = CheckpointLog::empty();
plugins.push_operation_change(change_entry("a"));
plugins.push_operation_change(change_entry("b"));
plugins.drain_operation_changes(&log);
assert_eq!(recorder.updated_ids(), [["a"], ["b"]]);
assert!(!plugins.is_draining());
}
#[test]
fn drained_change_holds_the_full_map() {
let recorder = test_support::Changes::default();
let plugins = test_support::plugins_of(recorder.clone());
let execution = OperationInfo::execution(
"e",
"EXECUTION",
&CheckpointStatus::Started,
None,
None,
None,
)
.unwrap();
assert!(plugins.replace_execution(execution));
let log = CheckpointLog::from_records(vec![(
"s".to_owned(),
record("STEP", CheckpointStatus::Started),
)]);
plugins.push_operation_change(change_entry("s"));
let before = crate::engine::projections();
plugins.drain_operation_changes(&log);
assert_eq!(crate::engine::projections() - before, 1);
let calls = recorder.calls();
let info = calls.first().unwrap();
assert_eq!(info.operations().keys().collect::<Vec<_>>(), ["e", "s"]);
assert_eq!(info.execution_arn(), plugins.info().execution_arn());
}
#[test]
fn replace_execution_reports_status_changes() {
let plugins = test_support::plugins_of(test_support::Changes::default());
let exec = |status: CheckpointStatus| {
OperationInfo::execution("e", "EXECUTION", &status, None, None, None).unwrap()
};
assert!(plugins.replace_execution(exec(CheckpointStatus::Started)));
assert!(!plugins.replace_execution(exec(CheckpointStatus::Started)));
assert!(plugins.replace_execution(exec(CheckpointStatus::Succeeded)));
}
#[test]
fn operations_map_inserts_execution_slot() {
let log = CheckpointLog::from_records(vec![(
"s".to_owned(),
record("STEP", CheckpointStatus::Started),
)]);
assert_eq!(operations_map(&log, None).len(), 1);
let execution = OperationInfo::execution(
"e",
"EXECUTION",
&CheckpointStatus::Started,
None,
None,
None,
);
let map = operations_map(&log, execution.as_ref());
assert_eq!(map.keys().collect::<Vec<_>>(), ["e", "s"]);
}
#[test]
fn attempt_outcome_wire_str_and_display() {
assert_eq!(AttemptOutcome::Succeeded.wire_str(), "SUCCEEDED");
assert_eq!(AttemptOutcome::Failed.wire_str(), "FAILED");
assert_eq!(AttemptOutcome::Succeeded.to_string(), "Succeeded");
assert_eq!(AttemptOutcome::Failed.to_string(), "Failed");
}
#[derive(Clone, Default)]
struct AttemptCapture(Arc<Mutex<Option<AttemptInfo>>>);
impl Plugin for AttemptCapture {
fn on_attempt_start(&self, info: &AttemptInfo) -> HookWork {
*self.0.lock().unwrap() = Some(info.clone());
Ok(None)
}
}
impl PluginFactory for AttemptCapture {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
#[test]
fn attempt_started_returns_the_attempt_infos_own_operation() {
let capture = AttemptCapture::default();
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
vec![("test::AttemptCapture", Arc::new(capture.clone()))];
let plugins = create_plugins(&factories, None, empty_info).expect("registered");
let operation = OperationInfo::from_record("a", &record("STEP", CheckpointStatus::Started));
let returned = attempt_started(&plugins, operation, 1, false);
let seen = capture.0.lock().unwrap().take().expect("the hook ran");
assert!(std::ptr::eq(seen.operation(), Arc::as_ptr(&returned)));
}
#[test]
fn attempt_started_keeps_the_records_attempt_in_the_start_hook() {
let hooks = test_support::OpHooks::default();
let plugins = hooks.plugins();
let mut started = record("STEP", CheckpointStatus::Started);
started.attempt = 2;
let operation = OperationInfo::from_record("a", &started);
assert_eq!(operation.attempt(), Some(2));
let returned = attempt_started(&plugins, operation, 3, true);
let seen: Vec<_> = hooks
.records()
.iter()
.map(|r| (r.hook, r.info_attempt, r.replay, r.status))
.collect();
assert_eq!(
seen,
[
("start", Some(2), true, Some(OperationStatus::Started)),
(
"attempt_start",
Some(3),
false,
Some(OperationStatus::Started)
),
]
);
assert_eq!(returned.attempt(), Some(3));
assert!(!returned.is_replay());
assert_eq!(returned.status(), Some(OperationStatus::Started));
assert!(returned.start_timestamp().is_some());
}
#[tokio::test]
async fn default_hooks_do_nothing() {
let operation = OperationInfo::from_record("a", &record("STEP", CheckpointStatus::Started));
let attempt = AttemptInfo {
operation: Arc::new(operation.clone()),
attempt: 1,
};
let attempt_end = AttemptEndInfo {
operation: operation.clone(),
attempt: 1,
outcome: AttemptOutcome::Succeeded,
error: None,
};
let change = OperationChangeInfo {
execution_arn: "arn".to_owned(),
updated_operations: Vec::new(),
operations: BTreeMap::new(),
};
let end = InvocationEndInfo {
request_id: "r".to_owned(),
execution_arn: "arn".to_owned(),
execution_input: "{}".to_owned(),
operations: BTreeMap::new(),
execution_start_timestamp: None,
status: InvocationStatus::Succeeded,
execution_result: None,
execution_error: None,
};
let propagation = PropagationInput {
execution_arn: "arn".to_owned(),
operation_id: "a".to_owned(),
parent_operation_id: None,
target_function_name: "f".to_owned(),
};
let info = empty_info();
assert!(Probe.on_invocation_start(&info).await.is_ok());
assert!(Probe.on_invocation_end(&end).await.is_ok());
assert!(Probe.on_operation_start(&operation).unwrap().is_none());
assert!(Probe.on_operation_end(&operation).unwrap().is_none());
assert!(Probe.on_attempt_start(&attempt).unwrap().is_none());
assert!(Probe.on_attempt_end(&attempt_end).unwrap().is_none());
assert!(Probe.on_operation_change(&change).unwrap().is_none());
assert_eq!(Probe.propagation_trace_id(&propagation).unwrap(), None);
assert_eq!(ProbeFactory(true).exclusive_group(), None);
}
use super::test_support::{Hooks, deadline_in, service, with_message};
use crate::DurableContext;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
fn start(
ctx: &DurableContext,
work: impl Future<Output = Result<(), BoxError>> + Send + 'static,
) {
ctx.plugins()
.expect("the invocation has plugins")
.start_work(0, "on_operation_end", Box::pin(work));
}
struct OnDrop(Option<Box<dyn FnOnce() + Send>>);
impl OnDrop {
fn new(f: impl FnOnce() + Send + 'static) -> Self {
Self(Some(Box::new(f)))
}
}
impl Drop for OnDrop {
fn drop(&mut self) {
if let Some(f) = self.0.take() {
f();
}
}
}
fn millis_since(start: tokio::time::Instant) -> u128 {
start.elapsed().as_millis()
}
const DEADLINE_ATTEMPTS: usize = 5;
const EXACT_SLACK_MS: u128 = 50;
fn slack(real: std::time::Instant) -> u128 {
real.elapsed().as_millis() + 2
}
fn retry(slack: u128, attempt: usize) -> bool {
slack >= EXACT_SLACK_MS && attempt + 1 < DEADLINE_ATTEMPTS
}
fn assert_at(what: &str, actual: u128, exact: u128, slack: u128) {
assert!(
actual <= exact && actual + slack >= exact,
"{what} at {actual} ms; expected {exact} ms, less at most {slack} ms"
);
}
type Shared<T> = Arc<Mutex<T>>;
fn shared<T: Default>() -> Shared<T> {
Arc::new(Mutex::new(T::default()))
}
fn end_at<F>(start: tokio::time::Instant, at: &Shared<Option<u128>>, then: F) -> Hooks
where
F: Fn() -> HookFuture<'static> + Send + Sync + 'static,
{
let at = Arc::clone(at);
Hooks::default().on_end(move |_| {
*at.lock().unwrap() = Some(millis_since(start));
then()
})
}
fn ready() -> HookFuture<'static> {
Box::pin(std::future::ready(Ok(())))
}
#[tokio::test(start_paused = true)]
async fn hook_work_runs_during_the_invocation() {
let flag = Arc::new(AtomicBool::new(false));
let seen = shared::<Option<bool>>();
let plugin = {
let flag = Arc::clone(&flag);
let seen = Arc::clone(&seen);
Hooks::default().on_end(move |_| {
*seen.lock().unwrap() = Some(flag.load(Ordering::SeqCst));
ready()
})
};
let (service, event) = service(vec![plugin], 0, move |_e, ctx| {
let flag = Arc::clone(&flag);
async move {
ctx.step(|_| async { Ok(1_u32) }).await?;
start(&ctx, async move {
flag.store(true, Ordering::SeqCst);
Ok(())
});
Ok(serde_json::Value::Null)
}
});
service(event).await.unwrap();
assert_eq!(*seen.lock().unwrap(), Some(true));
}
#[tokio::test(start_paused = true)]
async fn hook_work_does_not_delay_task_cleanup() {
for attempt in 0..DEADLINE_ATTEMPTS {
if hook_work_does_not_delay_task_cleanup_once(attempt).await {
return;
}
}
}
async fn hook_work_does_not_delay_task_cleanup_once(attempt: usize) -> bool {
let real = std::time::Instant::now();
let begin = tokio::time::Instant::now();
let observed = shared::<Option<u128>>();
let ended = shared::<Option<u128>>();
let end_started = shared::<Option<u128>>();
let plugin = end_at(begin, &end_started, ready);
let (service, event) = service(vec![plugin], deadline_in(2_000), {
let observed = Arc::clone(&observed);
let ended = Arc::clone(&ended);
move |_e, ctx| {
let observed = Arc::clone(&observed);
let ended = Arc::clone(&ended);
async move {
ctx.step(|_| async { Ok(1_u32) }).await?;
let probe = ctx.clone();
start(&ctx, async move {
let _guard = OnDrop::new(move || {
*ended.lock().unwrap() = Some(millis_since(begin));
});
while probe.coordinator().tasks().live() > 0 {
tokio::task::yield_now().await;
}
*observed.lock().unwrap() = Some(millis_since(begin));
drop(probe);
tokio::time::sleep(Duration::from_secs(20)).await;
Ok(())
});
Ok(serde_json::Value::Null)
}
}
});
service(event).await.unwrap();
let slack = slack(real);
if retry(slack, attempt) {
return false;
}
assert_eq!(*observed.lock().unwrap(), Some(0));
let aborted = ended
.lock()
.unwrap()
.expect("the work ended when the service returned");
assert_at("the drain aborted the work", aborted, 1_500, slack);
let end = end_started.lock().unwrap().expect("the end hook ran");
assert!(
end >= aborted,
"the end hook started at {end} ms, before the abort"
);
true
}
#[tokio::test(start_paused = true)]
async fn drain_waits_for_work_started_while_draining() {
let second_done = Arc::new(AtomicBool::new(false));
let seen = shared::<Option<bool>>();
let plugin = {
let second_done = Arc::clone(&second_done);
let seen = Arc::clone(&seen);
Hooks::default().on_end(move |_| {
*seen.lock().unwrap() = Some(second_done.load(Ordering::SeqCst));
ready()
})
};
let (service, event) = service(vec![plugin], 0, move |_e, ctx| {
let second_done = Arc::clone(&second_done);
async move {
let starter = ctx.clone();
start(&ctx, async move {
tokio::time::sleep(Duration::from_secs(1)).await;
start(&starter, async move {
tokio::time::sleep(Duration::from_secs(1)).await;
second_done.store(true, Ordering::SeqCst);
Ok(())
});
Ok(())
});
Ok(serde_json::Value::Null)
}
});
service(event).await.unwrap();
assert_eq!(*seen.lock().unwrap(), Some(true));
}
#[tokio::test(start_paused = true)]
async fn drain_aborts_work_started_while_waiting() {
for attempt in 0..DEADLINE_ATTEMPTS {
if drain_aborts_work_started_while_waiting_once(attempt).await {
return;
}
}
}
async fn drain_aborts_work_started_while_waiting_once(attempt: usize) -> bool {
let real = std::time::Instant::now();
let begin = tokio::time::Instant::now();
let ended = shared::<Option<u128>>();
let end_started = shared::<Option<u128>>();
let plugin = end_at(begin, &end_started, ready);
let (service, event) = service(vec![plugin], deadline_in(2_000), {
let ended = Arc::clone(&ended);
move |_e, ctx| {
let ended = Arc::clone(&ended);
async move {
let starter = ctx.clone();
start(&ctx, async move {
tokio::time::sleep(Duration::from_secs(20)).await;
Ok(())
});
start(&ctx, async move {
tokio::time::sleep(Duration::from_secs(1)).await;
start(&starter, async move {
let _guard = OnDrop::new(move || {
*ended.lock().unwrap() = Some(millis_since(begin));
});
tokio::time::sleep(Duration::from_secs(20)).await;
Ok(())
});
Ok(())
});
Ok(serde_json::Value::Null)
}
}
});
service(event).await.unwrap();
let slack = slack(real);
if retry(slack, attempt) {
return false;
}
let aborted = ended.lock().unwrap().expect("the second work ended");
assert_at("the drain aborted the second work", aborted, 1_500, slack);
let end = end_started.lock().unwrap().expect("the end hook ran");
assert!(end >= aborted, "the end hook started at {end} ms");
true
}
fn twenty_seconds_of_work(
begin: tokio::time::Instant,
ended: &Shared<Option<u128>>,
) -> impl Fn(
serde_json::Value,
DurableContext,
) -> Pin<Box<dyn Future<Output = Result<serde_json::Value, BoxError>> + Send>>
+ Send
+ Sync
+ 'static {
let ended = Arc::clone(ended);
move |_e, ctx| {
let ended = Arc::clone(&ended);
Box::pin(async move {
start(&ctx, async move {
let _guard = OnDrop::new(move || {
*ended.lock().unwrap() = Some(millis_since(begin));
});
tokio::time::sleep(Duration::from_secs(20)).await;
Ok(())
});
Ok(serde_json::Value::Null)
})
}
}
#[tokio::test(start_paused = true)]
#[expect(clippy::panic)] async fn drain_stops_500ms_and_end_50ms_before_deadline() {
for attempt in 0..DEADLINE_ATTEMPTS {
let (buffer, _guard) = capture();
let real = std::time::Instant::now();
let begin = tokio::time::Instant::now();
let ended = shared::<Option<u128>>();
let end_dropped = Arc::new(AtomicBool::new(false));
let plugin = {
let end_dropped = Arc::clone(&end_dropped);
Hooks::default().on_end(move |_| {
let guard = OnDrop::new({
let end_dropped = Arc::clone(&end_dropped);
move || end_dropped.store(true, Ordering::SeqCst)
});
Box::pin(async move {
let _guard = guard;
std::future::pending::<()>().await;
Ok(())
})
})
};
let (service, event) = service(
vec![plugin],
deadline_in(2_000),
twenty_seconds_of_work(begin, &ended),
);
service(event).await.unwrap();
let returned = millis_since(begin);
let slack = slack(real);
if retry(slack, attempt) {
continue;
}
assert_at("the service returned", returned, 1_950, slack);
let aborted = ended.lock().unwrap().expect("the work was aborted");
assert_at("the drain aborted the work", aborted, 1_500, slack);
assert!(
end_dropped.load(Ordering::SeqCst),
"the cut-off end hook was dropped"
);
let warnings = warnings(&buffer);
let drained = with_message(
&warnings,
"plugin hook work did not finish before the drain deadline; it is aborted",
);
let cut = with_message(
&warnings,
"plugin on_invocation_end hooks did not finish before the invocation deadline; \
the response returns without them",
);
let ([drained], [_]) = (drained.as_slice(), cut.as_slice()) else {
continue;
};
assert_eq!(drained.get("abandoned"), Some(&serde_json::json!(1)));
return;
}
panic!("no attempt captured both warnings");
}
#[tokio::test(start_paused = true)]
async fn end_hook_keeps_its_time() {
for attempt in 0..DEADLINE_ATTEMPTS {
if end_hook_keeps_its_time_once(attempt).await {
return;
}
}
}
async fn end_hook_keeps_its_time_once(attempt: usize) -> bool {
let real = std::time::Instant::now();
let begin = tokio::time::Instant::now();
let ended = shared::<Option<u128>>();
let finished = shared::<Option<u128>>();
let plugin = {
let finished = Arc::clone(&finished);
Hooks::default().on_end(move |_| {
let finished = Arc::clone(&finished);
Box::pin(async move {
tokio::time::sleep(Duration::from_millis(300)).await;
*finished.lock().unwrap() = Some(millis_since(begin));
Ok(())
})
})
};
let (service, event) = service(
vec![plugin],
deadline_in(2_000),
twenty_seconds_of_work(begin, &ended),
);
service(event).await.unwrap();
let slack = slack(real);
if retry(slack, attempt) {
return false;
}
let at = finished.lock().unwrap().expect("the end hook finished");
assert_at("the end hook finished", at, 1_800, slack);
true
}
#[tokio::test(start_paused = true)]
async fn drain_unbounded_without_deadline() {
let begin = tokio::time::Instant::now();
let ended = shared::<Option<u128>>();
let end_started = shared::<Option<u128>>();
let plugin = end_at(begin, &end_started, ready);
let (service, event) = service(vec![plugin], 0, twenty_seconds_of_work(begin, &ended));
service(event).await.unwrap();
assert_eq!(*ended.lock().unwrap(), Some(20_000));
assert_eq!(*end_started.lock().unwrap(), Some(20_000));
}
#[tokio::test(start_paused = true)]
async fn drain_skipped_inside_the_reserve() {
let begin = tokio::time::Instant::now();
let ended = shared::<Option<u128>>();
let end_started = shared::<Option<u128>>();
let plugin = end_at(begin, &end_started, ready);
let (service, event) = service(
vec![plugin],
deadline_in(400),
twenty_seconds_of_work(begin, &ended),
);
service(event).await.unwrap();
assert_eq!(*ended.lock().unwrap(), Some(0));
assert_eq!(*end_started.lock().unwrap(), Some(0));
}
#[derive(Debug)]
struct PanickingDisplayError;
impl fmt::Display for PanickingDisplayError {
#[expect(clippy::panic)] fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
panic!("display boom")
}
}
impl std::error::Error for PanickingDisplayError {}
#[tokio::test(start_paused = true)]
#[expect(clippy::panic)] async fn hook_work_errors_and_panics_are_logged() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let (service, event) = service(
vec![Hooks::default().named("Worker")],
0,
|_e, ctx| async move {
start(&ctx, async { Err("x".into()) });
start(&ctx, async { panic!("work boom") });
start(&ctx, async {
Err(Box::new(PanickingDisplayError) as BoxError)
});
Ok(serde_json::json!("out"))
},
);
let json = service(event).await.unwrap().0;
assert_eq!(json.get("Status"), Some(&serde_json::json!("SUCCEEDED")));
assert_eq!(json.get("Result"), Some(&serde_json::json!("\"out\"")));
let warnings = warnings(&buffer);
let failed = with_message(&warnings, "plugin hook failed; the error is ignored");
let panicked = with_message(&warnings, "plugin hook panicked; the panic is ignored");
let unformatted = with_message(
&warnings,
"plugin hook failed, and its error panicked while it was formatted; \
the error is ignored",
);
let ([failed], [panicked], [unformatted]) = (
failed.as_slice(),
panicked.as_slice(),
unformatted.as_slice(),
) else {
continue;
};
for warning in [failed, panicked, unformatted] {
assert_eq!(field(warning, "plugin"), "Worker (plugins[0])", "{warning}");
assert_eq!(field(warning, "hook"), "on_operation_end", "{warning}");
}
assert_eq!(field(failed, "error"), "x");
return;
}
panic!("no attempt captured the three warnings");
}
#[tokio::test(start_paused = true)]
#[expect(clippy::panic)] async fn hook_future_drop_panic_is_logged() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let plugin = Hooks::default().on_end(|_| {
let guard = OnDrop::new(|| panic!("future drop"));
Box::pin(async move {
let _guard = guard;
Ok(())
})
});
let (service, event) = service(vec![plugin], 0, |_e, _ctx| async move {
Ok(serde_json::Value::Null)
});
service(event).await.unwrap();
let warnings = warnings(&buffer);
let [warning] =
with_message(&warnings, "plugin hook panicked; the panic is ignored")[..]
else {
continue;
};
assert_eq!(field(warning, "hook"), "on_invocation_end");
return;
}
panic!("no attempt captured the warning");
}
#[derive(Clone, Default)]
struct UfPlugin {
start: Option<Arc<dyn Fn() -> HookWork + Send + Sync>>,
after_next: Option<Arc<dyn Fn() + Send + Sync>>,
}
impl UfPlugin {
fn starting(start: impl Fn() -> HookWork + Send + Sync + 'static) -> Self {
Self {
start: Some(Arc::new(start)),
..Self::default()
}
}
fn after_next(after: impl Fn() + Send + Sync + 'static) -> Self {
Self {
after_next: Some(Arc::new(after)),
..Self::default()
}
}
}
impl Plugin for UfPlugin {
fn name(&self) -> &'static str {
"UfPlugin"
}
fn on_user_function_start(&self, _info: &OperationInfo) -> HookWork {
self.start.as_ref().map_or(Ok(None), |start| start())
}
fn wrap_user_function(&self, _info: &OperationInfo, next: &mut dyn FnMut()) {
next();
if let Some(after) = &self.after_next {
after();
}
}
}
impl PluginFactory for UfPlugin {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
fn run_step_once(plugins: &Arc<InvocationPlugins>) -> Option<u32> {
let mut run = std::pin::pin!(run_user_function(
Some(Arc::clone(plugins)),
Arc::new(OperationInfo::for_test("s")),
UserBody::Step("s".to_owned()),
tracing::Span::none(),
|| async { Ok::<u32, BoxError>(7) },
|e: &BoxError| WireError::new(Some("Error".to_owned()), Some(e.to_string())),
));
let waker = std::task::Waker::noop();
let mut cx = Context::from_waker(waker);
match run.as_mut().poll(&mut cx) {
Poll::Ready(ran) => ran.into_parts().ok(),
Poll::Pending => None,
}
}
#[tokio::test]
async fn hook_future_drop_calls_wait_records_context_usage() {
let ctx = DurableContext::test_context();
let plugins = test_support::plugins_of(Hooks::default());
let dropped = Arc::new(AtomicBool::new(false));
let guard = OnDrop::new({
let ctx = ctx.clone();
let dropped = Arc::clone(&dropped);
move || {
drop(ctx.wait(Duration::from_secs(1)));
dropped.store(true, Ordering::SeqCst);
}
});
let mut run = HookRun {
identity: "Hooks (plugins[0])".into(),
hook: "on_user_function_start",
future: Some(Box::pin(async move {
let _guard = guard;
std::future::pending::<()>().await;
Ok(())
})),
undelivered: Arc::clone(&plugins.undelivered),
};
let waker = std::task::Waker::noop();
let mut cx = Context::from_waker(waker);
assert!(Pin::new(&mut run).poll(&mut cx).is_pending());
assert!(ctx.coordinator().fatal_error().is_none());
drop(run);
assert!(dropped.load(Ordering::SeqCst), "the destructor ran");
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert!(
fatal.error_message.contains("inside plugin code"),
"{}",
fatal.error_message
);
}
#[tokio::test]
async fn pending_hook_work_does_not_delay_suspension() {
#[derive(Clone)]
struct Gated {
gate: Arc<tokio::sync::Notify>,
polled: Arc<AtomicBool>,
done: Arc<AtomicBool>,
}
impl Plugin for Gated {
fn on_user_function_start(&self, _info: &OperationInfo) -> HookWork {
let (gate, polled, done) = (
Arc::clone(&self.gate),
Arc::clone(&self.polled),
Arc::clone(&self.done),
);
Ok(Some(Box::pin(async move {
polled.store(true, Ordering::SeqCst);
gate.notified().await;
done.store(true, Ordering::SeqCst);
Ok(())
})))
}
}
impl PluginFactory for Gated {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
let gated = Gated {
gate: Arc::new(tokio::sync::Notify::new()),
polled: Arc::default(),
done: Arc::default(),
};
let suspended = Arc::new(AtomicBool::new(false));
let (service, event) = test_support::service_configured(
{
let gated = gated.clone();
move |builder| builder.plugin(gated)
},
0,
Vec::new(),
{
let suspended = Arc::clone(&suspended);
move |_e, ctx| {
let suspended = Arc::clone(&suspended);
async move {
ctx.step(|_| async { Ok(1_u32) }).name("s").await?;
let _guard = OnDrop::new(move || suspended.store(true, Ordering::SeqCst));
ctx.wait(Duration::from_secs(1)).name("w").await?;
Ok(serde_json::json!("done"))
}
}
},
);
let mut invocation = std::pin::pin!(service(event));
let early = tokio::time::timeout(Duration::from_millis(200), invocation.as_mut()).await;
assert!(early.is_err(), "the drain waits for the hook future");
assert!(suspended.load(Ordering::SeqCst), "the invocation suspended");
assert!(gated.polled.load(Ordering::SeqCst), "the hook future ran");
assert!(
!gated.done.load(Ordering::SeqCst),
"the hook future is pending"
);
gated.gate.notify_one();
let json = tokio::time::timeout(Duration::from_secs(5), invocation)
.await
.expect("the drain ends once the gate opens")
.unwrap()
.0;
assert_eq!(
json.get("Status"),
Some(&serde_json::json!("PENDING")),
"{json}"
);
assert!(gated.done.load(Ordering::SeqCst));
}
#[derive(Debug)]
struct CountingDisplay {
calls: Arc<AtomicUsize>,
panics: bool,
}
impl fmt::Display for CountingDisplay {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.calls.fetch_add(1, Ordering::SeqCst);
assert!(!self.panics, "display boom");
f.write_str("counted")
}
}
impl std::error::Error for CountingDisplay {}
#[test]
#[expect(clippy::panic)] fn error_display_counted_once_and_formatting_panic_logged() {
'attempt: for _attempt in 0..5 {
for panics in [false, true] {
let calls = Arc::new(AtomicUsize::new(0));
let plugins = test_support::plugins_of(UfPlugin::starting({
let calls = Arc::clone(&calls);
move || {
Err(Box::new(CountingDisplay {
calls: Arc::clone(&calls),
panics,
}) as BoxError)
}
}));
let (buffer, guard) = capture();
let result = run_step_once(&plugins);
drop(guard);
assert_eq!(result, Some(7), "panics: {panics}");
assert_eq!(calls.load(Ordering::SeqCst), 1, "panics: {panics}");
let warnings = warnings(&buffer);
let [warning] = warnings.as_slice() else {
continue 'attempt;
};
assert_eq!(field(warning, "hook"), "on_user_function_start");
assert_eq!(field(warning, "plugin"), "UfPlugin (plugins[0])");
if panics {
assert_eq!(
field(warning, "message"),
"plugin hook failed, and its error panicked while it was formatted; \
the error is ignored"
);
assert!(warning.get("error").is_none(), "{warning}");
} else {
assert_eq!(
field(warning, "message"),
"plugin hook failed; the error is ignored"
);
assert_eq!(field(warning, "error"), "counted");
}
}
return;
}
panic!("no attempt captured exactly one warning per run");
}
#[tokio::test]
#[expect(clippy::panic)] async fn error_destructor_panic_fails_the_invocation() {
for _attempt in 0..5 {
let (buffer, guard) = capture();
let (service, event) = test_support::service_configured(
|builder| {
builder.plugin(UfPlugin::starting(|| {
Err(Box::new(DropPanicError) as BoxError)
}))
},
0,
Vec::new(),
|_e, ctx| async move {
let value = ctx.step(|_| async { Ok(7_u32) }).name("s").await?;
Ok(serde_json::json!(value))
},
);
let payload = panic_of(service(event)).await;
drop(guard);
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
let warnings = warnings(&buffer);
let [warning] = warnings.as_slice() else {
continue;
};
assert_eq!(
field(warning, "message"),
"plugin hook failed; the error is ignored"
);
assert_eq!(field(warning, "hook"), "on_user_function_start");
assert_eq!(field(warning, "error"), "drop err");
return;
}
panic!("no attempt captured exactly one warning");
}
#[test]
#[expect(clippy::panic)] fn panic_payload_destructor_panic_is_discarded() {
for _attempt in 0..5 {
#[expect(clippy::panic, reason = "the wrap panics with a payload")]
fn panic_with_payload() {
std::panic::panic_any(PanickingPayload)
}
let plugins = test_support::plugins_of(UfPlugin::after_next(panic_with_payload));
let (buffer, guard) = capture();
let result = run_step_once(&plugins);
drop(guard);
assert_eq!(result, Some(7));
let warnings = warnings(&buffer);
let [warning] = warnings.as_slice() else {
continue;
};
assert_eq!(
field(warning, "message"),
"plugin hook panicked; the panic is ignored"
);
assert_eq!(field(warning, "hook"), "wrap_user_function");
return;
}
panic!("no attempt captured exactly one warning");
}
#[test]
fn subscriber_panic_while_logging_is_discarded() {
struct Panicking(Arc<AtomicUsize>);
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Panicking {
#[expect(clippy::panic, reason = "the subscriber panics on every event")]
fn on_event(
&self,
_event: &tracing::Event<'_>,
_ctx: tracing_subscriber::layer::Context<'_, S>,
) {
self.0.fetch_add(1, Ordering::SeqCst);
panic!("subscriber boom");
}
}
#[expect(clippy::panic, reason = "the wrap panics after next()")]
fn wrap_boom() {
panic!("wrap boom")
}
use tracing_subscriber::layer::SubscriberExt as _;
let events = Arc::new(AtomicUsize::new(0));
let plugins = test_support::plugins_of(UfPlugin::after_next(wrap_boom));
let subscriber = tracing_subscriber::registry().with(Panicking(Arc::clone(&events)));
let result = {
let _guard = test_tracing::install_test_subscriber(subscriber);
run_step_once(&plugins)
};
assert_eq!(result, Some(7));
assert!(
events.load(Ordering::SeqCst) >= 1,
"the wrap panic was logged"
);
}
#[tokio::test(start_paused = true)]
#[expect(clippy::panic)] async fn work_started_after_the_drain_closed_is_dropped() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let escaped = shared::<Option<DurableContext>>();
let (service, event) = service(vec![Hooks::default()], 0, {
let escaped = Arc::clone(&escaped);
move |_e, ctx| {
let escaped = Arc::clone(&escaped);
async move {
*escaped.lock().unwrap() = Some(ctx);
Ok(serde_json::Value::Null)
}
}
});
service(event).await.unwrap();
let ctx = escaped.lock().unwrap().take().unwrap();
let polled = Arc::new(AtomicBool::new(false));
let dropped = Arc::new(AtomicBool::new(false));
let guard = OnDrop::new({
let dropped = Arc::clone(&dropped);
move || dropped.store(true, Ordering::SeqCst)
});
start(&ctx, {
let polled = Arc::clone(&polled);
async move {
let _guard = guard;
polled.store(true, Ordering::SeqCst);
Ok(())
}
});
tokio::task::yield_now().await;
assert!(!polled.load(Ordering::SeqCst));
assert!(dropped.load(Ordering::SeqCst));
let warnings = warnings(&buffer);
let [warning] = with_message(
&warnings,
"plugin hook work started after the drain closed; it is dropped",
)[..] else {
continue;
};
assert_eq!(field(warning, "hook"), "on_operation_end");
assert!(field(warning, "plugin").ends_with("(plugins[0])"));
return;
}
panic!("no attempt captured the warning");
}
#[test]
#[expect(clippy::panic)] fn work_without_a_runtime_is_dropped() {
for _attempt in 0..5 {
let (buffer, _guard) = capture();
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
vec![("Hooks", Arc::new(Hooks::default()))];
let plugins = create_plugins(&factories, None, empty_info).unwrap();
let dropped = Arc::new(AtomicBool::new(false));
let guard = OnDrop::new({
let dropped = Arc::clone(&dropped);
move || dropped.store(true, Ordering::SeqCst)
});
plugins.start_work(
0,
"on_operation_end",
Box::pin(async move {
let _guard = guard;
Ok(())
}),
);
assert!(dropped.load(Ordering::SeqCst));
let warnings = warnings(&buffer);
let [warning] = with_message(
&warnings,
"plugin hook work could not be spawned; it is dropped",
)[..] else {
continue;
};
assert_eq!(field(warning, "hook"), "on_operation_end");
return;
}
panic!("no attempt captured the warning");
}
#[tokio::test(start_paused = true)]
async fn start_work_removes_finished_handles() {
let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
vec![("Hooks", Arc::new(Hooks::default()))];
let plugins = create_plugins(&factories, None, empty_info).unwrap();
plugins.start_work(0, "on_operation_end", Box::pin(async { Ok(()) }));
tokio::time::sleep(Duration::from_millis(1)).await;
plugins.start_work(0, "on_operation_end", Box::pin(std::future::pending()));
let WorkState::Open(list) = &*plugins.lock_work() else {
unreachable!("the drain has not closed");
};
assert_eq!(list.len(), 1);
}
#[derive(Debug)]
struct DropPanicError;
impl fmt::Display for DropPanicError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("drop err")
}
}
impl std::error::Error for DropPanicError {}
impl Drop for DropPanicError {
#[expect(clippy::panic)] fn drop(&mut self) {
panic!("error drop");
}
}
#[derive(Debug)]
struct DropPanicDisplayError(&'static str);
impl fmt::Display for DropPanicDisplayError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.0)
}
}
impl std::error::Error for DropPanicDisplayError {}
impl Drop for DropPanicDisplayError {
#[expect(clippy::panic)] fn drop(&mut self) {
std::panic::panic_any(self.0);
}
}
#[expect(clippy::disallowed_methods)]
async fn panic_of<Fut>(future: Fut) -> Payload
where
Fut: Future + Send + 'static,
Fut::Output: Send,
{
match tokio::spawn(future).await {
Ok(_) => unreachable!("the service panicked"),
Err(join_err) => join_err.into_panic(),
}
}
#[tokio::test(start_paused = true)]
async fn start_hook_error_destructor_panic_fails_the_invocation() {
let plugin = Hooks::default()
.on_start(|_| Box::pin(async { Err(Box::new(DropPanicError) as BoxError) }));
let (service, event) = service(vec![plugin.clone()], 0, |_e, _ctx| async move {
Ok(serde_json::Value::Null)
});
let payload = panic_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
let ends = plugin.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Retrying);
}
#[tokio::test(start_paused = true)]
async fn hook_work_error_destructor_panic_fails_the_invocation() {
let (service, event) = service(vec![Hooks::default()], 0, |_e, ctx| async move {
start(&ctx, async { Err(Box::new(DropPanicError) as BoxError) });
std::future::pending::<()>().await;
Ok(serde_json::Value::Null)
});
let payload = panic_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
}
#[tokio::test(start_paused = true)]
async fn hook_work_error_destructor_panic_during_the_drain_fails_the_invocation() {
let plugin = Hooks::default();
let (service, event) = service(vec![plugin.clone()], 0, |_e, ctx| async move {
start(&ctx, async {
tokio::time::sleep(Duration::from_secs(1)).await;
Err(Box::new(DropPanicError) as BoxError)
});
Ok(serde_json::Value::Null)
});
let payload = panic_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
let ends = plugin.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Retrying);
assert_eq!(end.execution_result(), None);
let error = end.execution_error().expect("the panic's error");
assert_eq!(error.error_type(), None);
assert_eq!(error.error_message(), Some("error drop"));
}
#[tokio::test(start_paused = true)]
async fn end_hook_error_destructor_panic_fails_the_invocation() {
let failing = Hooks::default()
.on_end(|_| Box::pin(async { Err(Box::new(DropPanicError) as BoxError) }));
let healthy = Hooks::default();
let (service, event) = service(vec![failing, healthy.clone()], 0, |_e, _ctx| async move {
Ok(serde_json::json!("ok"))
});
let payload = panic_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
let ends = healthy.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Succeeded);
}
#[tokio::test(start_paused = true)]
async fn the_first_late_panic_resumes() {
let failing = Hooks::default().on_end(|_| {
Box::pin(async { Err(Box::new(DropPanicDisplayError("end drop")) as BoxError) })
});
let (service, event) = service(vec![failing], 0, |_e, ctx| async move {
start(&ctx, async {
tokio::time::sleep(Duration::from_secs(1)).await;
Err(Box::new(DropPanicDisplayError("work drop")) as BoxError)
});
Ok(serde_json::Value::Null)
});
let payload = panic_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"work drop"));
}
#[tokio::test(start_paused = true)]
#[expect(clippy::panic)] async fn late_panic_after_a_drive_panic_is_dropped() {
let failing = Hooks::default()
.on_end(|_| Box::pin(async { Err(Box::new(DropPanicError) as BoxError) }));
let (service, event) = service(vec![failing.clone()], 0, |_e, ctx| async move {
start(&ctx, async {
tokio::time::sleep(Duration::from_secs(1)).await;
Err(Box::new(DropPanicDisplayError("work drop")) as BoxError)
});
if ctx.is_replaying() {
return Ok(serde_json::Value::Null);
}
panic!("boom");
});
let payload = panic_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
let ends = failing.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info, got {}", ends.len());
};
assert_eq!(
end.execution_error().and_then(WireError::error_message),
Some("boom")
);
}
#[tokio::test(start_paused = true)]
async fn late_panic_after_a_fatal_error_is_dropped() {
let failing = Hooks::default()
.on_end(|_| Box::pin(async { Err(Box::new(DropPanicError) as BoxError) }));
let (service, event) = service(vec![failing.clone()], 0, |_e, ctx| async move {
ctx.coordinator()
.record_fatal("Error".to_owned(), "fatal".to_owned());
std::future::pending::<()>().await;
Ok(serde_json::Value::Null)
});
let response = service(event).await.expect("the FAILED envelope");
assert_eq!(
response.0.get("Status"),
Some(&serde_json::json!("FAILED")),
"{:?}",
response.0
);
let ends = failing.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Failed);
}
struct Checked<F> {
inner: Pin<Box<F>>,
ended: Arc<AtomicBool>,
violations: Arc<AtomicUsize>,
_live: OnDrop,
}
impl<F: Future<Output = Result<(), BoxError>>> Future for Checked<F> {
type Output = Result<(), BoxError>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
if self.ended.load(Ordering::SeqCst) {
self.violations.fetch_add(1, Ordering::SeqCst);
}
self.inner.as_mut().poll(cx)
}
}
#[derive(Clone)]
struct Round {
ended: Arc<AtomicBool>,
violations: Arc<AtomicUsize>,
live: Arc<AtomicUsize>,
}
impl Round {
fn work(&self, ctx: DurableContext, depth: u32) -> HookFuture<'static> {
self.live.fetch_add(1, Ordering::SeqCst);
let live = Arc::clone(&self.live);
let round = self.clone();
Box::pin(Checked {
inner: Box::pin(async move {
tokio::time::sleep(Duration::from_millis(1)).await;
if depth > 0 {
let next = round.work(ctx.clone(), depth - 1);
ctx.plugins()
.unwrap()
.start_work(0, "on_operation_end", next);
}
Ok(())
}),
ended: Arc::clone(&self.ended),
violations: Arc::clone(&self.violations),
_live: OnDrop::new(move || {
live.fetch_sub(1, Ordering::SeqCst);
}),
})
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[expect(clippy::disallowed_methods)]
async fn late_start_racing_the_drain_never_runs_after_end_starts() {
for round_index in 0..20 {
let round = Round {
ended: Arc::new(AtomicBool::new(false)),
violations: Arc::new(AtomicUsize::new(0)),
live: Arc::new(AtomicUsize::new(0)),
};
let plugin = {
let ended = Arc::clone(&round.ended);
Hooks::default().on_end(move |_| {
ended.store(true, Ordering::SeqCst);
ready()
})
};
let starters = shared::<Vec<JoinHandle<()>>>();
let (service, event) = service(vec![plugin], deadline_in(700), {
let round = round.clone();
let starters = Arc::clone(&starters);
move |_e, ctx| {
let round = round.clone();
let starters = Arc::clone(&starters);
async move {
for _ in 0..4 {
ctx.plugins().unwrap().start_work(
0,
"on_operation_end",
round.work(ctx.clone(), 5),
);
}
for _ in 0..3 {
let ctx = ctx.clone();
let round = round.clone();
starters.lock().unwrap().push(tokio::spawn(async move {
let mut after_end: Option<std::time::Instant> = None;
loop {
let work = round.work(ctx.clone(), 1);
ctx.plugins()
.unwrap()
.start_work(0, "on_operation_end", work);
if round.ended.load(Ordering::SeqCst) {
let since =
*after_end.get_or_insert_with(std::time::Instant::now);
if since.elapsed() > Duration::from_millis(20) {
break;
}
}
tokio::task::yield_now().await;
}
}));
}
Ok(serde_json::Value::Null)
}
}
});
service(event).await.unwrap();
let handles = std::mem::take(&mut *starters.lock().unwrap());
for handle in handles {
handle.await.unwrap();
}
assert_eq!(
round.violations.load(Ordering::SeqCst),
0,
"round {round_index}"
);
assert_eq!(round.live.load(Ordering::SeqCst), 0, "round {round_index}");
}
}
struct DropFlag(Arc<AtomicBool>);
impl Drop for DropFlag {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
fn one_plugin() -> Arc<InvocationPlugins> {
let log = Arc::new(Mutex::new(Vec::new()));
create_plugins(
&factories(vec![("A", Behavior::Some)], &log),
None,
empty_info,
)
.unwrap()
}
#[expect(clippy::panic)] fn pending_work_dropping(
flag: Arc<AtomicBool>,
) -> impl Future<Output = Result<(), BoxError>> + Send + 'static {
let guard = OnDrop::new({
move || {
flag.store(true, Ordering::SeqCst);
std::panic::panic_any(PanickingPayload);
}
});
async move {
let _guard = guard;
std::future::pending::<Result<(), BoxError>>().await
}
}
async fn settle() {
for _ in 0..10 {
tokio::task::yield_now().await;
}
}
#[tokio::test(start_paused = true)]
async fn dropping_the_plugins_aborts_hook_work() {
let dropped = Arc::new(AtomicBool::new(false));
let plugins = one_plugin();
plugins.start_work(
0,
"on_operation_end",
Box::pin(pending_work_dropping(Arc::clone(&dropped))),
);
settle().await;
assert!(!dropped.load(Ordering::SeqCst), "the work runs");
drop(plugins);
settle().await;
assert!(dropped.load(Ordering::SeqCst), "the work was aborted");
}
#[tokio::test(start_paused = true)]
async fn cancelled_invocation_aborts_hook_work() {
let dropped = Arc::new(AtomicBool::new(false));
let escaped = shared::<Option<DurableContext>>();
let (service, event) = service(vec![Hooks::default()], 0, {
let dropped = Arc::clone(&dropped);
let escaped = Arc::clone(&escaped);
move |_e, ctx| {
let dropped = Arc::clone(&dropped);
let escaped = Arc::clone(&escaped);
async move {
start(&ctx, pending_work_dropping(dropped));
*escaped.lock().unwrap() = Some(ctx);
std::future::pending::<()>().await;
Ok(serde_json::Value::Null)
}
}
});
let cancelled = tokio::time::timeout(Duration::from_secs(1), service(event)).await;
assert!(cancelled.is_err(), "the invocation was cancelled");
settle().await;
let ctx = escaped.lock().unwrap().take().expect("the handler ran");
assert!(
ctx.plugins().is_some(),
"the escaped context keeps the plugins"
);
assert!(dropped.load(Ordering::SeqCst), "the work was aborted");
}
#[tokio::test(start_paused = true)]
async fn cancelled_drain_aborts_hook_work() {
let dropped = Arc::new(AtomicBool::new(false));
let escaped = shared::<Option<DurableContext>>();
let (service, event) = service(vec![Hooks::default()], 0, {
let dropped = Arc::clone(&dropped);
let escaped = Arc::clone(&escaped);
move |_e, ctx| {
let dropped = Arc::clone(&dropped);
let escaped = Arc::clone(&escaped);
async move {
start(&ctx, pending_work_dropping(dropped));
*escaped.lock().unwrap() = Some(ctx);
Ok(serde_json::Value::Null)
}
}
});
let cancelled = tokio::time::timeout(Duration::from_secs(1), service(event)).await;
assert!(cancelled.is_err(), "the drain waited until the cancel");
settle().await;
let ctx = escaped.lock().unwrap().take().expect("the handler ran");
assert!(
ctx.plugins().is_some(),
"the escaped context keeps the plugins"
);
assert!(dropped.load(Ordering::SeqCst), "the work was aborted");
}
#[test]
fn panic_recorded_after_shut_down_is_dropped() {
let plugins = one_plugin();
plugins.shut_down();
let dropped = Arc::new(AtomicBool::new(false));
Undelivered::record(
&plugins.undelivered,
Box::new(DropFlag(Arc::clone(&dropped))),
);
assert!(
dropped.load(Ordering::SeqCst),
"the payload was dropped at once"
);
let nested = catch_unwind(AssertUnwindSafe(|| {
Undelivered::record(&plugins.undelivered, Box::new(PanickingPayload));
}));
assert!(
nested.is_ok(),
"a nested payload's destructor panic escaped"
);
assert!(plugins.take_error_drop_panics().is_empty());
}
#[test]
fn late_panic_after_shut_down_is_dropped() {
let state = Mutex::new(Undelivered::Discarding);
let dropped = Arc::new(AtomicBool::new(false));
Undelivered::keep_late(&state, Box::new(DropFlag(Arc::clone(&dropped))));
assert!(
dropped.load(Ordering::SeqCst),
"the payload was dropped at once"
);
let nested = catch_unwind(AssertUnwindSafe(|| {
Undelivered::keep_late(&state, Box::new(PanickingPayload));
}));
assert!(
nested.is_ok(),
"a nested payload's destructor panic escaped"
);
}
struct HoldsPlugins(
#[expect(dead_code, reason = "held only to be dropped")] Arc<InvocationPlugins>,
);
#[test]
fn late_panic_holding_the_plugins_is_dropped_after_the_lock() {
let (done, finished) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let plugins = one_plugin();
let undelivered = Arc::clone(&plugins.undelivered);
plugins.shut_down();
Undelivered::keep_late(&undelivered, Box::new(HoldsPlugins(plugins)));
let _ = done.send(());
});
assert!(
finished.recv_timeout(Duration::from_secs(30)).is_ok(),
"dropping the payload deadlocked on the state's lock"
);
}
#[test]
fn shut_down_drops_kept_panics() {
let log = Arc::new(Mutex::new(Vec::new()));
let factories = factories(vec![("bad", Behavior::ErrorPanickingNestedDrop)], &log);
let plugins = create_plugins(&factories, None, empty_info).unwrap();
let shut = catch_unwind(AssertUnwindSafe(|| plugins.shut_down()));
assert!(shut.is_ok(), "a nested payload's destructor panic escaped");
assert!(plugins.take_error_drop_panics().is_empty());
}
use super::test_support::{TraceHook, trace_plugins};
const CONFLICT: &str = "conflicting propagation trace ids; keeping the first value";
const TRACE_PANICKED: &str = "plugin propagation_trace_id panicked; the value is ignored";
const TRACE_FAILED: &str = "plugin propagation_trace_id failed; the value is ignored";
fn trace_input() -> PropagationInput {
PropagationInput::new(
"arn:exec".to_owned(),
"op-1".to_owned(),
Some("parent-1".to_owned()),
"target-fn".to_owned(),
)
}
#[expect(clippy::panic)] fn trace_id_of(
hooks: &[TraceHook],
expected: usize,
) -> (Option<String>, Vec<serde_json::Value>) {
for _attempt in 0..5 {
let (buffer, guard) = capture();
let value = trace_plugins(hooks).propagation_trace_id(&trace_input());
drop(guard);
let warnings = warnings(&buffer);
if warnings.len() == expected {
return (value, warnings);
}
assert!(
warnings.len() < expected,
"more warnings than expected: {warnings:?}"
);
}
panic!("no attempt captured {expected} warnings");
}
#[test]
fn trace_id_none_without_answers() {
let (value, warnings) = trace_id_of(&[TraceHook::value(None), TraceHook::value(None)], 0);
assert_eq!(value, None);
assert!(warnings.is_empty(), "{warnings:?}");
}
#[test]
fn trace_id_none_continues_to_later_plugins() {
let hooks = [
TraceHook::value(None),
TraceHook::value(Some("Root=1-a")),
TraceHook::value(None),
TraceHook::value(Some("Root=1-b")),
];
let (value, warnings) = trace_id_of(&hooks, 1);
assert_eq!(value.as_deref(), Some("Root=1-a"));
let calls: Vec<usize> = hooks.iter().map(|hook| hook.inputs().len()).collect();
let first = calls.first().copied().unwrap_or(0);
assert!(first > 0, "{calls:?}");
assert!(calls.iter().all(|&count| count == first), "{calls:?}");
let [conflict] = with_message(&warnings, CONFLICT)[..] else {
unreachable!("one conflict: {warnings:?}");
};
assert_eq!(field(conflict, "first_plugin"), "TraceHook (plugins[1])");
assert_eq!(field(conflict, "later_plugin"), "TraceHook (plugins[3])");
assert_eq!(conflict.get("conflict_count"), Some(&serde_json::json!(1)));
}
#[test]
fn trace_id_first_non_blank_wins() {
let (value, warnings) = trace_id_of(
&[
TraceHook::value(Some("Root=1-a")),
TraceHook::value(Some("Root=1-b")),
],
1,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
let [conflict] = with_message(&warnings, CONFLICT)[..] else {
unreachable!("one conflict: {warnings:?}");
};
assert_eq!(field(conflict, "field"), "x_amzn_trace_id");
assert!(
field(conflict, "first_plugin").ends_with("(plugins[0])"),
"{conflict}"
);
assert!(
field(conflict, "later_plugin").ends_with("(plugins[1])"),
"{conflict}"
);
assert_eq!(conflict.get("conflict_count"), Some(&serde_json::json!(1)));
}
#[test]
fn trace_id_conflicts_are_counted() {
let (value, warnings) = trace_id_of(
&[
TraceHook::value(Some("Root=1-a")),
TraceHook::value(Some("Root=1-b")),
TraceHook::value(Some("Root=1-c")),
],
2,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
let conflicts = with_message(&warnings, CONFLICT);
let counts: Vec<_> = conflicts
.iter()
.map(|w| {
(
w.get("conflict_count").cloned(),
field(w, "later_plugin").to_owned(),
)
})
.collect();
assert_eq!(
counts,
[
(
Some(serde_json::json!(1)),
"TraceHook (plugins[1])".to_owned()
),
(
Some(serde_json::json!(2)),
"TraceHook (plugins[2])".to_owned()
),
]
);
for conflict in conflicts {
assert_eq!(field(conflict, "first_plugin"), "TraceHook (plugins[0])");
}
}
#[test]
fn trace_id_blank_is_skipped() {
let (value, warnings) = trace_id_of(
&[
TraceHook::value(Some(" ")),
TraceHook::value(Some("Root=1-a")),
],
0,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
assert!(warnings.is_empty(), "{warnings:?}");
}
#[test]
fn trace_id_blank_after_value_is_no_conflict() {
let (value, warnings) = trace_id_of(
&[
TraceHook::value(Some("Root=1-a")),
TraceHook::value(Some("")),
],
0,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
assert!(warnings.is_empty(), "{warnings:?}");
}
#[test]
fn trace_id_same_value_is_no_conflict() {
let (value, warnings) = trace_id_of(
&[
TraceHook::value(Some("Root=1-a")),
TraceHook::value(Some("Root=1-a")),
],
0,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
assert!(warnings.is_empty(), "{warnings:?}");
}
#[test]
fn trace_id_kept_unchanged() {
let (value, warnings) = trace_id_of(&[TraceHook::value(Some(" Root=1-a "))], 0);
assert_eq!(value.as_deref(), Some(" Root=1-a "));
assert!(warnings.is_empty(), "{warnings:?}");
}
#[test]
#[expect(clippy::panic)] fn trace_id_panic_is_ignored() {
let (value, warnings) = trace_id_of(
&[
TraceHook::new(|_| panic!("trace boom")),
TraceHook::value(Some("Root=1-a")),
],
1,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
let [panicked] = with_message(&warnings, TRACE_PANICKED)[..] else {
unreachable!("one panic record: {warnings:?}");
};
assert!(
field(panicked, "plugin").ends_with("(plugins[0])"),
"{panicked}"
);
assert_eq!(field(panicked, "hook"), "propagation_trace_id");
}
#[test]
fn propagation_err_is_logged_and_skipped() {
let (value, warnings) = trace_id_of(
&[
TraceHook::new(|_| Err("x".into())),
TraceHook::value(Some("Root=1-a")),
],
1,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
let [failed] = with_message(&warnings, TRACE_FAILED)[..] else {
unreachable!("one failure record: {warnings:?}");
};
assert!(
field(failed, "plugin").ends_with("(plugins[0])"),
"{failed}"
);
assert_eq!(field(failed, "error"), "x");
assert_eq!(field(failed, "hook"), "propagation_trace_id");
}
#[test]
fn propagation_err_display_panic_is_logged() {
let (value, warnings) = trace_id_of(
&[
TraceHook::new(|_| Err(Box::new(PanickingDisplayError))),
TraceHook::value(Some("Root=1-a")),
],
1,
);
assert_eq!(value.as_deref(), Some("Root=1-a"));
let [unformatted] = with_message(
&warnings,
"plugin propagation_trace_id failed, and its error panicked while it was \
formatted; the value is ignored",
)[..] else {
unreachable!("one record: {warnings:?}");
};
assert_eq!(field(unformatted, "plugin"), "TraceHook (plugins[0])");
assert_eq!(field(unformatted, "hook"), "propagation_trace_id");
}
#[test]
fn trace_id_hook_calling_an_operation_fails_the_execution() {
let ctx = DurableContext::test_context();
let captured = ctx.clone();
let plugins = trace_plugins(&[TraceHook::new(move |_| {
drop(captured.claim_operation_id("wait"));
Ok(None)
})]);
assert_eq!(plugins.propagation_trace_id(&trace_input()), None);
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert!(
fatal.error_message.contains("inside plugin code"),
"{}",
fatal.error_message
);
}
#[test]
fn propagation_err_destructor_panic_is_recorded() {
let plugins = trace_plugins(&[
TraceHook::new(|_| Err(Box::new(DropPanicError))),
TraceHook::value(Some("Root=1-a")),
]);
let value = plugins.propagation_trace_id(&trace_input());
assert_eq!(value.as_deref(), Some("Root=1-a"));
let kept = plugins.take_error_drop_panics();
let [payload] = kept.as_slice() else {
unreachable!("one kept panic, got {}", kept.len());
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
}
#[test]
fn trace_id_input_fields() {
let hook = TraceHook::value(None);
let value = trace_plugins(std::slice::from_ref(&hook)).propagation_trace_id(&trace_input());
assert_eq!(value, None);
let inputs = hook.inputs();
let [input] = inputs.as_slice() else {
unreachable!("one call, got {}", inputs.len());
};
assert_eq!(input.execution_arn(), "arn:exec");
assert_eq!(input.operation_id(), "op-1");
assert_eq!(input.parent_operation_id(), Some("parent-1"));
assert_eq!(input.target_function_name(), "target-fn");
}
}