use std::collections::BTreeMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::task::Waker;
use std::time::{Duration, SystemTime};
use tokio::time::Instant;
use crate::driver::{FatalError, FatalSeverity};
use crate::engine::CheckpointStatus;
use crate::task::TaskTracker;
const INITIAL_REFRESH_DELAY: Duration = Duration::from_secs(1);
const MIN_REMAINING_TO_REFRESH: Duration = Duration::from_secs(1);
const REFRESH_INTERVAL_CEILING: Duration = Duration::from_secs(10);
const EXTENDED_REFRESH_INTERVAL: Duration = Duration::from_secs(60);
const REFRESHES_BEFORE_EXTENDED_INTERVAL: u32 = 95;
fn fits_before_deadline(delay: Duration, remaining: Option<Duration>) -> bool {
remaining.is_none_or(|r| delay.saturating_add(MIN_REMAINING_TO_REFRESH) <= r)
}
fn first_refresh_delay(
due_at: Option<SystemTime>,
now: SystemTime,
remaining: Option<Duration>,
) -> Option<Duration> {
let delay = due_at.map_or(INITIAL_REFRESH_DELAY, |due| {
due.duration_since(now).unwrap_or(Duration::ZERO)
});
fits_before_deadline(delay, remaining).then_some(delay)
}
fn refresh_delay(unchanged: u32) -> Duration {
if unchanged > REFRESHES_BEFORE_EXTENDED_INTERVAL {
EXTENDED_REFRESH_INTERVAL
} else {
Duration::from_secs(u64::from(unchanged)).min(REFRESH_INTERVAL_CEILING)
}
}
pub(crate) const WITHDRAWAL_WARNING: &str = "Checkpoint response contained no CheckpointToken: \
the service will accept no further checkpoints from this invocation. Suspending; the \
execution continues on the next invocation.";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Phase {
Running,
Suspending,
Finished,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum PermitKind {
Activity,
Checkpoint,
Scheduler,
}
#[derive(Debug)]
struct Registration {
waker: Waker,
due_at: Option<SystemTime>,
status: CheckpointStatus,
awaited: Option<Arc<AtomicBool>>,
armed: bool,
next_refresh: Option<Instant>,
unchanged: u32,
restart: bool,
}
impl Registration {
fn is_awaited(&self) -> bool {
self.awaited
.as_ref()
.is_none_or(|flag| flag.load(Ordering::SeqCst))
}
fn arm(&mut self, now: Instant, system_now: SystemTime, remaining: Option<Duration>) {
self.armed = true;
self.next_refresh =
first_refresh_delay(self.due_at, system_now, remaining).map(|delay| now + delay);
}
}
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum DueRefresh {
Nothing,
UnderFloor,
Due {
keys: Vec<(String, u64)>,
first_wire_id: String,
},
}
#[derive(Debug)]
struct State {
phase: Phase,
activity: usize,
checkpoint: usize,
scheduler: usize,
waits: BTreeMap<(String, u64), Registration>,
next_seq: u64,
fatal: Option<FatalError>,
panic: Option<Box<dyn std::any::Any + Send + 'static>>,
panic_open: bool,
driver: Option<Waker>,
deadline: Option<Instant>,
withdrawn: bool,
write_failure: Option<FatalError>,
#[cfg(test)]
wake_log: Vec<String>,
}
impl State {
fn check_suspension(&mut self) -> Option<Waker> {
let quiescent = self.activity == 0 && self.checkpoint == 0 && self.scheduler == 0;
if self.phase == Phase::Running
&& !self.withdrawn
&& quiescent
&& !self.waits.is_empty()
&& self.fatal.is_none()
&& self.panic.is_none()
{
self.phase = Phase::Suspending;
return self.driver.clone();
}
None
}
fn admits(&self) -> bool {
self.phase == Phase::Running && !self.withdrawn
}
fn window_preempted(&self) -> bool {
self.phase != Phase::Running || (self.activity == 0 && !self.waits.is_empty())
}
fn remaining(&self, now: Instant) -> Option<Duration> {
self.deadline
.map(|deadline| deadline.saturating_duration_since(now))
}
}
#[derive(Debug)]
pub(crate) struct InvocationCoordinator {
state: Mutex<State>,
tasks: Arc<TaskTracker>,
window: tokio::sync::Notify,
rearm: tokio::sync::Notify,
clock: (Instant, SystemTime),
}
impl InvocationCoordinator {
pub(crate) fn new() -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(State {
phase: Phase::Running,
activity: 0,
checkpoint: 0,
scheduler: 0,
waits: BTreeMap::new(),
next_seq: 0,
fatal: None,
panic: None,
panic_open: true,
driver: None,
deadline: None,
withdrawn: false,
write_failure: None,
#[cfg(test)]
wake_log: Vec::new(),
}),
tasks: Arc::new(TaskTracker::new()),
window: tokio::sync::Notify::new(),
rearm: tokio::sync::Notify::new(),
clock: (Instant::now(), SystemTime::now()),
})
}
fn system_time_at(&self, now: Instant) -> SystemTime {
let (anchor, anchor_system) = self.clock;
anchor_system + now.saturating_duration_since(anchor)
}
pub(crate) fn start_deadline(&self, remaining: Option<Duration>) {
let now = Instant::now();
self.lock().deadline = remaining.map(|d| now + d);
}
fn lock(&self) -> MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
pub(crate) fn acquire(self: &Arc<Self>, kind: PermitKind) -> Option<Permit> {
let mut state = self.lock();
if !state.admits() {
return None;
}
match kind {
PermitKind::Activity => state.activity += 1,
PermitKind::Checkpoint => {
if state.fatal.is_some() {
return None;
}
state.checkpoint += 1;
}
PermitKind::Scheduler => {
if state.activity == 0 && state.checkpoint == 0 {
return None;
}
state.scheduler += 1;
}
}
drop(state);
Some(Permit {
coordinator: Arc::clone(self),
kind,
})
}
pub(crate) fn register_wait(
self: &Arc<Self>,
wire_id: &str,
due_at: Option<SystemTime>,
status: &CheckpointStatus,
awaited: Option<Arc<AtomicBool>>,
waker: &Waker,
) -> Option<WaitRegistration> {
let now = Instant::now();
let mut state = self.lock();
if !state.admits() {
return None;
}
let mut registration = Registration {
waker: waker.clone(),
due_at,
status: status.clone(),
awaited,
armed: false,
next_refresh: None,
unchanged: 0,
restart: false,
};
if registration.is_awaited() {
registration.arm(now, self.system_time_at(now), state.remaining(now));
}
let armed = registration.next_refresh.is_some();
let key = (wire_id.to_owned(), state.next_seq);
state.next_seq += 1;
state.waits.insert(key.clone(), registration);
drop(state);
if armed {
self.rearm.notify_one();
}
Some(WaitRegistration {
coordinator: Arc::clone(self),
key,
})
}
#[cfg(test)]
pub(crate) fn register_test_wait(
self: &Arc<Self>,
wire_id: &str,
waker: &Waker,
) -> Option<WaitRegistration> {
self.register_wait(wire_id, None, &CheckpointStatus::Started, None, waker)
}
pub(crate) fn notify_awaited(&self) {
let now = Instant::now();
let armed = {
let mut state = self.lock();
let remaining = state.remaining(now);
let system_now = self.system_time_at(now);
let mut armed = false;
for registration in state.waits.values_mut() {
if !registration.armed && registration.is_awaited() {
registration.arm(now, system_now, remaining);
armed |= registration.next_refresh.is_some();
}
}
armed
};
if armed {
self.rearm.notify_one();
}
}
pub(crate) fn next_refresh_at(&self) -> Option<Instant> {
self.lock()
.waits
.values()
.filter_map(|registration| registration.next_refresh)
.min()
}
pub(crate) async fn rearmed(&self) {
self.rearm.notified().await;
}
pub(crate) fn take_due(&self, now: Instant) -> DueRefresh {
let mut state = self.lock();
let under_floor = state
.remaining(now)
.is_some_and(|r| r < MIN_REMAINING_TO_REFRESH);
let mut keys = Vec::new();
for (key, registration) in &mut state.waits {
if registration.next_refresh.is_some_and(|at| at <= now) {
if under_floor {
registration.next_refresh = None;
}
keys.push(key.clone());
}
}
match keys.first() {
None => DueRefresh::Nothing,
Some(_) if under_floor => DueRefresh::UnderFloor,
Some((first_wire_id, _)) => DueRefresh::Due {
first_wire_id: first_wire_id.clone(),
keys,
},
}
}
pub(crate) fn after_refresh(
&self,
keys: &[(String, u64)],
records: &BTreeMap<String, (CheckpointStatus, Option<SystemTime>)>,
) {
let now = Instant::now();
let mut changed: Vec<Waker> = Vec::new();
{
let mut state = self.lock();
let remaining = state.remaining(now);
for key in keys {
let Some(registration) = state.waits.get_mut(key) else {
continue;
};
match records.get(&key.0) {
Some((status, due_at)) if *status != registration.status => {
registration.status = status.clone();
registration.due_at = *due_at;
registration.unchanged = 0;
registration.next_refresh = None;
registration.restart = true;
changed.push(registration.waker.clone());
}
_ => {
registration.unchanged = registration.unchanged.saturating_add(1);
let delay = refresh_delay(registration.unchanged);
registration.next_refresh =
fits_before_deadline(delay, remaining).then(|| now + delay);
}
}
}
}
for waker in changed {
waker.wake();
}
}
pub(crate) fn wake_operations(&self, wire_ids: &[String]) {
if wire_ids.is_empty() {
return;
}
let wakers: Vec<Waker> = {
#[cfg_attr(not(test), expect(unused_mut))]
let mut state = self.lock();
let woken: Vec<(String, Waker)> = state
.waits
.iter()
.filter(|((id, _), _)| wire_ids.iter().any(|wanted| wanted == id))
.map(|((id, _), registration)| (id.clone(), registration.waker.clone()))
.collect();
#[cfg(test)]
state
.wake_log
.extend(woken.iter().map(|(id, _)| id.clone()));
woken.into_iter().map(|(_, waker)| waker).collect()
};
for waker in wakers {
waker.wake();
}
}
#[cfg(test)]
pub(crate) fn wake_log(&self) -> Vec<String> {
self.lock().wake_log.clone()
}
pub(crate) fn phase(&self) -> Phase {
self.lock().phase
}
#[cfg(test)]
pub(crate) fn live_waits(&self) -> usize {
self.lock().waits.len()
}
#[cfg(test)]
pub(crate) fn permits(&self, kind: PermitKind) -> usize {
let state = self.lock();
match kind {
PermitKind::Activity => state.activity,
PermitKind::Checkpoint => state.checkpoint,
PermitKind::Scheduler => state.scheduler,
}
}
pub(crate) fn tasks(&self) -> &Arc<TaskTracker> {
&self.tasks
}
pub(crate) fn finish(&self) {
self.lock().phase = Phase::Finished;
self.window.notify_waiters();
}
pub(crate) async fn coalescing_window(&self, delay: Duration) {
let mut sleep = std::pin::pin!(tokio::time::sleep(delay));
loop {
let mut notified = std::pin::pin!(self.window.notified());
notified.as_mut().enable();
if self.lock().window_preempted() {
return;
}
tokio::select! {
biased;
() = &mut sleep => return,
() = notified => {}
}
}
}
pub(crate) fn withdraw(
&self,
execution_arn: &str,
request_id: &str,
drop_queued: impl FnOnce(),
) {
{
let mut state = self.lock();
if state.withdrawn {
return;
}
state.withdrawn = true;
}
tracing::warn!(
executionArn = %execution_arn,
requestId = %request_id,
"{WITHDRAWAL_WARNING}"
);
drop_queued();
let driver = {
let mut state = self.lock();
if state.phase == Phase::Running {
state.phase = Phase::Suspending;
}
state.driver.clone()
};
self.after_change((driver, true));
}
pub(crate) fn fail_write(&self, fatal: FatalError, drop_queued: impl FnOnce()) {
let driver = {
let mut state = self.lock();
if state.write_failure.is_none() {
state.write_failure = Some(fatal.clone());
}
if state.fatal.is_none() {
state.fatal = Some(fatal);
}
state.driver.clone()
};
drop_queued();
if let Some(driver) = driver {
driver.wake();
}
}
pub(crate) fn write_failed(&self) -> bool {
self.lock().write_failure.is_some()
}
pub(crate) fn write_failure(&self) -> Option<FatalError> {
self.lock().write_failure.clone()
}
pub(crate) fn withdrawn(&self) -> bool {
self.lock().withdrawn
}
pub(crate) fn record_fatal(&self, error_type: String, error_message: String) {
self.record(FatalError {
error_type,
error_message,
severity: FatalSeverity::FailExecution,
});
}
pub(crate) fn record_invocation_fault(&self, error_type: &'static str, error_message: String) {
self.record(FatalError {
error_type: error_type.to_owned(),
error_message,
severity: FatalSeverity::FailInvocation,
});
}
pub(crate) fn record(&self, fatal: FatalError) {
let driver = {
let mut state = self.lock();
if state.fatal.is_none() {
state.fatal = Some(fatal);
}
state.driver.clone()
};
if let Some(driver) = driver {
driver.wake();
}
}
pub(crate) fn record_panic(
&self,
payload: Box<dyn std::any::Any + Send + 'static>,
) -> Option<Box<dyn std::any::Any + Send + 'static>> {
let (driver, rejected) = {
let mut state = self.lock();
let rejected = if state.panic_open && state.panic.is_none() {
state.panic = Some(payload);
None
} else {
Some(payload)
};
(state.driver.clone(), rejected)
};
if let Some(driver) = driver {
driver.wake();
}
rejected
}
pub(crate) fn take_panic(&self) -> Option<Box<dyn std::any::Any + Send + 'static>> {
let mut state = self.lock();
state.panic_open = false;
state.panic.take()
}
pub(crate) fn has_panic(&self) -> bool {
self.lock().panic.is_some()
}
pub(crate) fn fatal_error(&self) -> Option<FatalError> {
self.lock().fatal.clone()
}
pub(crate) fn register_driver_waker(&self, waker: &Waker) {
let mut state = self.lock();
match &state.driver {
Some(existing) if existing.will_wake(waker) => {}
_ => state.driver = Some(waker.clone()),
}
}
pub(crate) fn running_and_idle(&self) -> bool {
let state = self.lock();
state.phase == Phase::Running
&& state.activity == 0
&& state.checkpoint == 0
&& state.scheduler == 0
&& state.waits.is_empty()
}
fn release(&self, kind: PermitKind) {
let driver = {
let mut state = self.lock();
match kind {
PermitKind::Activity => state.activity = state.activity.saturating_sub(1),
PermitKind::Checkpoint => state.checkpoint = state.checkpoint.saturating_sub(1),
PermitKind::Scheduler => state.scheduler = state.scheduler.saturating_sub(1),
}
(state.check_suspension(), state.window_preempted())
};
self.after_change(driver);
}
fn after_change(&self, (driver, preempted): (Option<Waker>, bool)) {
if let Some(driver) = driver {
driver.wake();
}
if preempted {
self.window.notify_waiters();
}
}
fn retire(&self, key: &(String, u64)) {
let (removed, driver) = {
let mut state = self.lock();
let removed = state.waits.remove(key);
(removed, state.check_suspension())
};
let was_armed = removed
.as_ref()
.is_some_and(|registration| registration.next_refresh.is_some());
drop(removed);
if was_armed {
self.rearm.notify_one();
}
self.after_change((driver, false));
}
}
#[derive(Debug)]
pub(crate) struct Permit {
coordinator: Arc<InvocationCoordinator>,
kind: PermitKind,
}
impl Drop for Permit {
fn drop(&mut self) {
self.coordinator.release(self.kind);
}
}
#[derive(Debug)]
pub(crate) struct WaitRegistration {
coordinator: Arc<InvocationCoordinator>,
key: (String, u64),
}
impl WaitRegistration {
pub(crate) fn update_waker(&self, waker: &Waker) {
let old = {
let mut state = self.coordinator.lock();
match state.waits.get_mut(&self.key) {
Some(existing) if !existing.waker.will_wake(waker) => {
Some(std::mem::replace(&mut existing.waker, waker.clone()))
}
_ => None,
}
};
drop(old);
}
}
impl WaitRegistration {
pub(crate) fn resume_schedule(&self) {
let now = Instant::now();
let armed = {
let mut state = self.coordinator.lock();
let remaining = state.remaining(now);
let system_now = self.coordinator.system_time_at(now);
match state.waits.get_mut(&self.key) {
Some(registration) if registration.restart => {
registration.restart = false;
registration.arm(now, system_now, remaining);
registration.next_refresh.is_some()
}
_ => false,
}
};
if armed {
self.coordinator.rearm.notify_one();
}
}
}
impl Drop for WaitRegistration {
fn drop(&mut self) {
self.coordinator.retire(&self.key);
}
}
#[cfg(test)]
pub(crate) mod race_points {
use std::cell::RefCell;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum RacePoint {
DirectQueueLock,
BufferLock,
DirectNoRuntimeDrain,
BatchNoRuntimeClaim,
}
type Hook = Box<dyn FnMut(RacePoint)>;
thread_local! {
static HOOK: RefCell<Option<Hook>> = const { RefCell::new(None) };
}
pub(crate) fn set(hook: impl FnMut(RacePoint) + 'static) {
HOOK.with(|slot| *slot.borrow_mut() = Some(Box::new(hook)));
}
pub(crate) fn reach(point: RacePoint) {
let hook = HOOK.with(|slot| slot.borrow_mut().take());
if let Some(mut hook) = hook {
hook(point);
HOOK.with(|slot| {
let mut slot = slot.borrow_mut();
if slot.is_none() {
*slot = Some(hook);
}
});
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::task::{Wake, Waker};
use super::{
DueRefresh, InvocationCoordinator, PermitKind, Phase, first_refresh_delay, refresh_delay,
};
use crate::driver::{FatalError, FatalSeverity};
use crate::engine::CheckpointStatus;
use crate::task::test_support::{Schedule, ScheduleState};
use std::collections::BTreeMap;
use std::sync::atomic::AtomicBool;
use std::time::{Duration, SystemTime};
struct CountingWaker(AtomicUsize);
impl Wake for CountingWaker {
fn wake(self: Arc<Self>) {
self.0.fetch_add(1, Ordering::SeqCst);
}
}
fn counting_waker() -> (Arc<CountingWaker>, Waker) {
let inner = Arc::new(CountingWaker(AtomicUsize::new(0)));
(Arc::clone(&inner), Waker::from(inner))
}
fn activity(coordinator: &InvocationCoordinator) -> usize {
coordinator.permits(PermitKind::Activity)
}
#[tokio::test(start_paused = true)]
async fn coalescing_window_runs_full_delay_without_wait() {
let coordinator = InvocationCoordinator::new();
let started = tokio::time::Instant::now();
coordinator.coalescing_window(Duration::from_secs(10)).await;
assert_eq!(started.elapsed(), Duration::from_secs(10));
}
#[tokio::test(start_paused = true)]
#[expect(clippy::disallowed_methods)]
async fn coalescing_window_rechecks_after_notification() {
let coordinator = InvocationCoordinator::new();
let _buffered = coordinator
.acquire(PermitKind::Checkpoint)
.expect("running");
let first = coordinator.acquire(PermitKind::Activity).expect("running");
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
let window_coordinator = Arc::clone(&coordinator);
let ended = Arc::new(AtomicUsize::new(0));
let window_ended = Arc::clone(&ended);
let window = tokio::spawn(async move {
window_coordinator
.coalescing_window(Duration::from_secs(3600))
.await;
window_ended.store(1, Ordering::SeqCst);
});
tokio::task::yield_now().await;
drop(first);
let second = coordinator.acquire(PermitKind::Activity).expect("running");
tokio::task::yield_now().await;
assert_eq!(ended.load(Ordering::SeqCst), 0, "the window waits on");
let started = tokio::time::Instant::now();
drop(second);
window.await.expect("the window task does not panic");
assert_eq!(ended.load(Ordering::SeqCst), 1);
assert!(
started.elapsed() < Duration::from_secs(1),
"the release with a live wait ends the window"
);
}
#[test]
fn first_wake_takes_one_permit() {
let coordinator = InvocationCoordinator::new();
let schedule = Schedule::idle(&coordinator);
let waker = schedule.waker();
waker.wake_by_ref();
assert_eq!(activity(&coordinator), 1);
assert_eq!(schedule.state(), ScheduleState::Queued);
waker.wake_by_ref();
assert_eq!(activity(&coordinator), 1, "a duplicate wake takes nothing");
assert_eq!(schedule.state(), ScheduleState::Queued);
}
#[test]
fn wake_during_poll_keeps_permit() {
let coordinator = InvocationCoordinator::new();
let schedule = Schedule::idle(&coordinator);
assert!(schedule.begin_poll());
assert_eq!(schedule.state(), ScheduleState::Polling);
schedule.waker().wake_by_ref();
schedule.end_pending(false);
assert_eq!(schedule.state(), ScheduleState::Queued);
assert_eq!(activity(&coordinator), 1);
let other = Schedule::idle(&coordinator);
drop(schedule);
assert_eq!(activity(&coordinator), 0);
assert!(other.begin_poll());
other.end_pending(false);
assert_eq!(other.state(), ScheduleState::Idle);
assert_eq!(activity(&coordinator), 0);
}
#[test]
fn completion_wakes_consumer_before_release() {
let coordinator = InvocationCoordinator::new();
let consumer = Schedule::idle(&coordinator);
let finished = Schedule::idle(&coordinator);
let slot = crate::task::test_support::slot_with_waker::<u32>(&consumer.waker());
assert!(finished.begin_poll());
assert_eq!(activity(&coordinator), 1);
let permit = finished.end_ready();
slot.deliver_value(7);
assert_eq!(consumer.state(), ScheduleState::Queued);
assert_eq!(activity(&coordinator), 2);
drop(permit);
assert_eq!(activity(&coordinator), 1);
assert_eq!(consumer.state(), ScheduleState::Queued);
}
#[test]
fn suspends_at_zero_with_a_live_wait() {
let coordinator = InvocationCoordinator::new();
let (driver_count, driver) = counting_waker();
coordinator.register_driver_waker(&driver);
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
let permit = coordinator.acquire(PermitKind::Activity).expect("running");
drop(permit);
assert_eq!(coordinator.phase(), Phase::Suspending);
assert_eq!(driver_count.0.load(Ordering::SeqCst), 1);
assert!(coordinator.acquire(PermitKind::Activity).is_none());
assert!(coordinator.acquire(PermitKind::Checkpoint).is_none());
assert!(
coordinator
.register_test_wait("op", Waker::noop())
.is_none()
);
}
#[test]
fn no_wait_no_suspension() {
let coordinator = InvocationCoordinator::new();
let permit = coordinator.acquire(PermitKind::Activity).expect("running");
drop(permit);
assert_eq!(coordinator.phase(), Phase::Running);
}
#[test]
fn held_permits_block_suspension() {
let coordinator = InvocationCoordinator::new();
let wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
let activity = coordinator.acquire(PermitKind::Activity).expect("running");
let checkpoint = coordinator
.acquire(PermitKind::Checkpoint)
.expect("running");
drop(activity);
assert_eq!(coordinator.phase(), Phase::Running);
drop(checkpoint);
assert_eq!(coordinator.phase(), Phase::Suspending);
drop(wait);
assert_eq!(coordinator.phase(), Phase::Suspending);
}
#[test]
fn fatal_blocks_suspension() {
let coordinator = InvocationCoordinator::new();
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
let permit = coordinator.acquire(PermitKind::Activity).expect("running");
coordinator.record_fatal("Error".to_owned(), "boom".to_owned());
drop(permit);
assert_eq!(coordinator.phase(), Phase::Running);
}
#[test]
fn fatal_record_refuses_checkpoint_permits() {
type Record = fn(&InvocationCoordinator);
let records: [(&str, Record); 3] = [
("record_fatal", |c| {
c.record_fatal("Error".to_owned(), "boom".to_owned());
}),
("record_invocation_fault", |c| {
c.record_invocation_fault("Error", "boom".to_owned());
}),
("record", |c| {
c.record(write_fatal("Usage", FatalSeverity::FailExecution));
}),
];
for (name, record) in records {
let coordinator = InvocationCoordinator::new();
let held = coordinator
.acquire(PermitKind::Checkpoint)
.expect("running");
record(&coordinator);
assert_eq!(
coordinator.lock().checkpoint,
1,
"{name}: the permit stays held"
);
assert!(
coordinator.acquire(PermitKind::Checkpoint).is_none(),
"{name}: a later checkpoint permit is refused"
);
assert_eq!(coordinator.lock().checkpoint, 1, "{name}");
let activity = coordinator.acquire(PermitKind::Activity);
assert!(activity.is_some(), "{name}: an activity permit is granted");
drop(activity);
drop(held);
assert_eq!(coordinator.lock().checkpoint, 0, "{name}");
}
}
#[test]
fn panic_blocks_suspension() {
let coordinator = InvocationCoordinator::new();
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
let permit = coordinator.acquire(PermitKind::Activity).expect("running");
assert!(coordinator.record_panic(Box::new("boom")).is_none());
drop(permit);
assert_eq!(coordinator.phase(), Phase::Running);
}
#[test]
fn first_panic_is_kept() {
let coordinator = InvocationCoordinator::new();
assert!(!coordinator.has_panic());
assert!(coordinator.record_panic(Box::new("first")).is_none());
assert!(coordinator.has_panic());
let second = coordinator
.record_panic(Box::new("second"))
.expect("the second payload is returned");
assert_eq!(second.downcast_ref::<&str>(), Some(&"second"));
let kept = coordinator.take_panic().expect("kept");
assert_eq!(kept.downcast_ref::<&str>(), Some(&"first"));
assert!(!coordinator.has_panic());
}
#[test]
fn record_after_take_returns_the_payload() {
let coordinator = InvocationCoordinator::new();
assert!(coordinator.take_panic().is_none());
let late = coordinator
.record_panic(Box::new("late"))
.expect("ownership is closed");
assert_eq!(late.downcast_ref::<&str>(), Some(&"late"));
assert!(coordinator.take_panic().is_none());
}
struct LockProbe {
coordinator: std::sync::Weak<InvocationCoordinator>,
woken_unlocked: AtomicUsize,
woken_locked: AtomicUsize,
}
impl Wake for LockProbe {
fn wake(self: Arc<Self>) {
self.wake_by_ref();
}
fn wake_by_ref(self: &Arc<Self>) {
let coordinator = self.coordinator.upgrade().expect("the test holds it");
if coordinator.state.try_lock().is_ok() {
self.woken_unlocked.fetch_add(1, Ordering::SeqCst);
} else {
self.woken_locked.fetch_add(1, Ordering::SeqCst);
}
}
}
#[test]
fn kept_panic_wakes_driver() {
let coordinator = InvocationCoordinator::new();
let probe = Arc::new(LockProbe {
coordinator: Arc::downgrade(&coordinator),
woken_unlocked: AtomicUsize::new(0),
woken_locked: AtomicUsize::new(0),
});
coordinator.register_driver_waker(&Waker::from(Arc::clone(&probe)));
assert!(coordinator.record_panic(Box::new("boom")).is_none());
assert_eq!(probe.woken_unlocked.load(Ordering::SeqCst), 1);
assert_eq!(probe.woken_locked.load(Ordering::SeqCst), 0);
}
#[test]
fn retired_last_wait_does_not_suspend() {
let coordinator = InvocationCoordinator::new();
let wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
drop(wait);
assert_eq!(coordinator.phase(), Phase::Running);
assert_eq!(coordinator.live_waits(), 0);
}
#[test]
fn wake_operations_wakes_only_named_registrations() {
let coordinator = InvocationCoordinator::new();
let (a_count, a) = counting_waker();
let (b_count, b) = counting_waker();
let _a = coordinator.register_test_wait("a", &a).expect("running");
let _b = coordinator.register_test_wait("b", &b).expect("running");
coordinator.wake_operations(&["a".to_owned()]);
assert_eq!(a_count.0.load(Ordering::SeqCst), 1);
assert_eq!(b_count.0.load(Ordering::SeqCst), 0);
coordinator.wake_operations(&[]);
assert_eq!(a_count.0.load(Ordering::SeqCst), 1);
}
#[test]
fn update_waker_replaces_the_registered_waker() {
let coordinator = InvocationCoordinator::new();
let (a_count, a) = counting_waker();
let (b_count, b) = counting_waker();
let registration = coordinator.register_test_wait("op", &a).expect("running");
registration.update_waker(&b);
coordinator.wake_operations(&["op".to_owned()]);
assert_eq!(a_count.0.load(Ordering::SeqCst), 0);
assert_eq!(b_count.0.load(Ordering::SeqCst), 1);
registration.update_waker(&b);
coordinator.wake_operations(&["op".to_owned()]);
assert_eq!(b_count.0.load(Ordering::SeqCst), 2);
}
#[test]
fn finish_refuses_permits_and_registrations() {
let coordinator = InvocationCoordinator::new();
coordinator.finish();
assert_eq!(coordinator.phase(), Phase::Finished);
assert!(coordinator.acquire(PermitKind::Activity).is_none());
assert!(
coordinator
.register_test_wait("op", Waker::noop())
.is_none()
);
}
#[test]
fn running_and_idle_reads_phase_and_counts_together() {
let coordinator = InvocationCoordinator::new();
assert!(coordinator.running_and_idle());
let permit = coordinator.acquire(PermitKind::Activity).expect("running");
assert!(!coordinator.running_and_idle());
let checkpoint = coordinator
.acquire(PermitKind::Checkpoint)
.expect("running");
drop(permit);
assert!(!coordinator.running_and_idle());
drop(checkpoint);
let registration = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
assert!(!coordinator.running_and_idle());
drop(registration);
assert!(coordinator.running_and_idle());
let root = coordinator.acquire(PermitKind::Activity).expect("running");
coordinator.finish();
drop(root);
assert_eq!(coordinator.permits(PermitKind::Activity), 0);
assert_eq!(coordinator.live_waits(), 0);
assert!(!coordinator.running_and_idle());
}
#[test]
fn invocation_fault_is_recorded_once() {
let coordinator = InvocationCoordinator::new();
coordinator.record_invocation_fault("FirstError", "first".to_owned());
coordinator.record_fatal("Error".to_owned(), "second".to_owned());
let fatal = coordinator.fatal_error().expect("recorded");
assert_eq!(fatal.error_message, "first");
assert_eq!(fatal.severity, FatalSeverity::FailInvocation);
}
#[test]
fn withdraw_latches_once_and_suspends() {
let coordinator = InvocationCoordinator::new();
let (counter, waker) = counting_waker();
coordinator.register_driver_waker(&waker);
let _permit = coordinator.acquire(PermitKind::Activity).expect("running");
assert!(!coordinator.withdrawn());
coordinator.withdraw("arn", "req", || {});
assert!(coordinator.withdrawn());
assert_eq!(coordinator.phase(), Phase::Suspending);
assert_eq!(counter.0.load(Ordering::SeqCst), 1);
coordinator.withdraw("arn", "req", || {});
assert_eq!(
counter.0.load(Ordering::SeqCst),
1,
"a second withdrawal does nothing"
);
}
#[test]
fn withdraw_closes_admission_before_dropping_and_suspends_after() {
let coordinator = InvocationCoordinator::new();
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
let permit = std::cell::RefCell::new(Some(
coordinator
.acquire(PermitKind::Checkpoint)
.expect("running"),
));
let drops = std::cell::Cell::new(0);
coordinator.withdraw("arn", "req", || {
drops.set(drops.get() + 1);
assert!(coordinator.withdrawn(), "the latch is set first");
assert!(
coordinator.acquire(PermitKind::Activity).is_none(),
"no activity permit after the latch"
);
assert!(
coordinator.acquire(PermitKind::Checkpoint).is_none(),
"no checkpoint permit after the latch"
);
assert!(
coordinator.acquire(PermitKind::Scheduler).is_none(),
"no scheduler permit after the latch"
);
assert!(
coordinator
.register_test_wait("other", Waker::noop())
.is_none(),
"no wait registration after the latch"
);
drop(permit.borrow_mut().take());
assert_eq!(coordinator.phase(), Phase::Running, "the phase moves last");
});
assert_eq!(drops.get(), 1);
assert_eq!(coordinator.phase(), Phase::Suspending);
coordinator.withdraw("arn", "req", || drops.set(drops.get() + 1));
assert_eq!(drops.get(), 1, "a second withdrawal drops nothing");
}
#[test]
fn withdraw_after_finish_keeps_finished() {
let coordinator = InvocationCoordinator::new();
coordinator.finish();
coordinator.withdraw("arn", "req", || {});
assert!(coordinator.withdrawn());
assert_eq!(coordinator.phase(), Phase::Finished);
}
fn write_fatal(name: &str, severity: FatalSeverity) -> FatalError {
FatalError {
error_type: name.to_owned(),
error_message: name.to_lowercase(),
severity,
}
}
#[test]
fn fail_write_latches_records_drops_and_wakes() {
let coordinator = InvocationCoordinator::new();
let (counter, waker) = counting_waker();
coordinator.register_driver_waker(&waker);
assert!(!coordinator.write_failed());
let drops = std::cell::Cell::new(0);
coordinator.fail_write(write_fatal("First", FatalSeverity::FailInvocation), || {
drops.set(drops.get() + 1);
});
assert!(coordinator.write_failed());
assert_eq!(drops.get(), 1);
assert_eq!(counter.0.load(Ordering::SeqCst), 1);
assert_eq!(
coordinator.fatal_error().expect("recorded").error_type,
"First"
);
coordinator.fail_write(write_fatal("Second", FatalSeverity::FailExecution), || {});
assert_eq!(
coordinator.write_failure().expect("latched").error_type,
"First",
"the first failure keeps the latch"
);
assert_eq!(
coordinator.fatal_error().expect("recorded").error_type,
"First"
);
}
#[test]
fn fail_write_after_another_fatal_keeps_the_fatal_slot() {
let coordinator = InvocationCoordinator::new();
coordinator.record(write_fatal("Usage", FatalSeverity::FailExecution));
coordinator.fail_write(write_fatal("Write", FatalSeverity::FailExecution), || {});
assert_eq!(
coordinator.fatal_error().expect("recorded").error_type,
"Usage"
);
assert_eq!(
coordinator.write_failure().expect("latched").error_type,
"Write"
);
}
#[test]
fn wake_precedes_checkpoint_release() {
let coordinator = InvocationCoordinator::new();
let schedule = Schedule::idle(&coordinator);
let registration = coordinator
.register_test_wait("op", &schedule.waker())
.expect("running");
let checkpoint = coordinator
.acquire(PermitKind::Checkpoint)
.expect("running");
let log = crate::engine::CheckpointLog::empty();
log.insert(
"op".to_owned(),
crate::engine::CheckpointRecord::with_status("op", CheckpointStatus::Succeeded),
);
coordinator.wake_operations(&["op".to_owned()]);
drop(checkpoint);
assert_eq!(schedule.state(), ScheduleState::Queued);
assert_eq!(activity(&coordinator), 1);
assert_eq!(coordinator.phase(), Phase::Running);
drop(registration);
}
#[tokio::test]
async fn checkpoint_write_after_suspension_is_rejected() {
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = crate::context::DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
);
let coordinator = Arc::clone(ctx.coordinator());
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
drop(coordinator.acquire(PermitKind::Activity));
assert_eq!(coordinator.phase(), Phase::Suspending);
let update = aws_sdk_lambda::types::OperationUpdate::builder()
.id("u")
.r#type(aws_sdk_lambda::types::OperationType::Step)
.action(aws_sdk_lambda::types::OperationAction::Start)
.build()
.expect("all required fields set");
let written =
tokio::time::timeout(Duration::from_secs(1), ctx.checkpoint_updates(vec![update]))
.await;
assert!(written.is_err(), "the rejected write never resolves");
assert!(client.recorded_updates().is_empty(), "nothing was sent");
}
#[test]
fn refresh_delay_table() {
let table = [(1, 1), (2, 2), (10, 10), (11, 10), (95, 10), (96, 60)];
for (unchanged, expected) in table {
assert_eq!(
refresh_delay(unchanged),
Duration::from_secs(expected),
"after {unchanged} unchanged refreshes"
);
}
}
#[test]
fn first_refresh_rules() {
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
let due = Some(now + Duration::from_secs(5));
let secs = Duration::from_secs;
assert_eq!(first_refresh_delay(due, now, None), Some(secs(5)));
assert_eq!(
first_refresh_delay(due, now, Some(Duration::from_millis(5_500))),
None
);
assert_eq!(first_refresh_delay(due, now, Some(secs(6))), Some(secs(5)));
assert_eq!(first_refresh_delay(None, now, None), Some(secs(1)));
assert_eq!(
first_refresh_delay(None, now, Some(Duration::from_millis(1_500))),
None
);
assert_eq!(
first_refresh_delay(Some(now - secs(2)), now, None),
Some(Duration::ZERO)
);
}
#[test]
fn scheduler_permit_needs_other_local_work() {
let coordinator = InvocationCoordinator::new();
assert!(
coordinator.acquire(PermitKind::Scheduler).is_none(),
"no other permit is held"
);
let activity = coordinator.acquire(PermitKind::Activity).expect("running");
let with_activity = coordinator.acquire(PermitKind::Scheduler);
assert!(with_activity.is_some(), "beside an activity permit");
drop(activity);
drop(with_activity);
let checkpoint = coordinator
.acquire(PermitKind::Checkpoint)
.expect("running");
let with_checkpoint = coordinator.acquire(PermitKind::Scheduler);
assert!(with_checkpoint.is_some(), "beside a checkpoint permit");
coordinator.finish();
assert!(
coordinator.acquire(PermitKind::Scheduler).is_none(),
"after the drive ended"
);
drop((checkpoint, with_checkpoint));
}
#[test]
fn scheduler_permit_blocks_suspension() {
let coordinator = InvocationCoordinator::new();
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
let activity = coordinator.acquire(PermitKind::Activity).expect("running");
let scheduler = coordinator
.acquire(PermitKind::Scheduler)
.expect("beside work");
drop(activity);
assert_eq!(coordinator.phase(), Phase::Running);
assert!(!coordinator.running_and_idle());
drop(scheduler);
assert_eq!(coordinator.phase(), Phase::Suspending);
}
#[tokio::test(start_paused = true)]
async fn unawaited_registration_is_armed_by_notify_awaited() {
let coordinator = InvocationCoordinator::new();
let flag = Arc::new(AtomicBool::new(false));
let _wait = coordinator
.register_wait(
"op",
None,
&CheckpointStatus::Started,
Some(Arc::clone(&flag)),
Waker::noop(),
)
.expect("running");
assert_eq!(coordinator.next_refresh_at(), None, "not awaited yet");
coordinator.notify_awaited();
assert_eq!(coordinator.next_refresh_at(), None, "the flag is not set");
flag.store(true, Ordering::SeqCst);
let polled = tokio::time::Instant::now();
coordinator.notify_awaited();
assert_eq!(
coordinator.next_refresh_at(),
Some(polled + Duration::from_secs(1))
);
}
#[tokio::test(start_paused = true)]
async fn due_registration_under_the_floor_is_not_armed_again() {
let coordinator = InvocationCoordinator::new();
coordinator.start_deadline(Some(Duration::from_secs(3)));
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
assert_eq!(
coordinator.take_due(tokio::time::Instant::now()),
DueRefresh::Nothing
);
tokio::time::advance(Duration::from_millis(2_500)).await;
assert_eq!(
coordinator.take_due(tokio::time::Instant::now()),
DueRefresh::UnderFloor
);
assert_eq!(coordinator.next_refresh_at(), None);
}
#[tokio::test(start_paused = true)]
async fn after_refresh_backs_off_restarts_and_skips_retired() {
let coordinator = InvocationCoordinator::new();
let (woken, waker) = counting_waker();
let unchanged = coordinator
.register_test_wait("same", Waker::noop())
.expect("running");
let changed = coordinator
.register_test_wait("changed", &waker)
.expect("running");
let retired = coordinator
.register_test_wait("retired", Waker::noop())
.expect("running");
tokio::time::advance(Duration::from_secs(1)).await;
let (keys, first_wire_id) = match coordinator.take_due(tokio::time::Instant::now()) {
DueRefresh::Due {
keys,
first_wire_id,
} => (keys, first_wire_id),
other => unreachable!("three registrations are due, got {other:?}"),
};
assert_eq!(keys.len(), 3);
assert_eq!(first_wire_id, "changed", "key order");
drop(retired);
let records = BTreeMap::from([
("same".to_owned(), (CheckpointStatus::Started, None)),
("changed".to_owned(), (CheckpointStatus::Pending, None)),
]);
coordinator.after_refresh(&keys, &records);
let after = tokio::time::Instant::now();
assert_eq!(
woken.0.load(Ordering::SeqCst),
1,
"the changed operation is woken"
);
assert_eq!(
coordinator.next_refresh_at(),
Some(after + Duration::from_secs(1))
);
changed.resume_schedule();
assert_eq!(
coordinator.next_refresh_at(),
Some(after + Duration::from_secs(1))
);
drop(unchanged);
assert_eq!(
coordinator.next_refresh_at(),
Some(after + Duration::from_secs(1)),
"the restarted schedule is due 1 s after the restart"
);
tokio::time::advance(Duration::from_millis(500)).await;
changed.resume_schedule();
assert_eq!(
coordinator.next_refresh_at(),
Some(after + Duration::from_secs(1))
);
}
#[tokio::test(start_paused = true)]
async fn after_refresh_stops_arming_near_the_deadline() {
let coordinator = InvocationCoordinator::new();
coordinator.start_deadline(Some(Duration::from_millis(2_500)));
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
tokio::time::advance(Duration::from_secs(1)).await;
let DueRefresh::Due { keys, .. } = coordinator.take_due(tokio::time::Instant::now()) else {
unreachable!("the registration is due");
};
let records = BTreeMap::from([("op".to_owned(), (CheckpointStatus::Started, None))]);
coordinator.after_refresh(&keys, &records);
assert_eq!(coordinator.next_refresh_at(), None);
}
#[tokio::test(start_paused = true)]
async fn notify_awaited_keeps_an_armed_registration() {
let coordinator = InvocationCoordinator::new();
let registered = tokio::time::Instant::now();
let _wait = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
tokio::time::advance(Duration::from_millis(500)).await;
coordinator.notify_awaited();
assert_eq!(
coordinator.next_refresh_at(),
Some(registered + Duration::from_secs(1))
);
}
#[test]
fn scheduler_permit_is_not_idle() {
let coordinator = InvocationCoordinator::new();
let activity = coordinator.acquire(PermitKind::Activity).expect("running");
let scheduler = coordinator
.acquire(PermitKind::Scheduler)
.expect("beside work");
drop(activity);
assert!(!coordinator.running_and_idle());
assert_eq!(coordinator.permits(PermitKind::Scheduler), 1);
drop(scheduler);
assert_eq!(coordinator.permits(PermitKind::Scheduler), 0);
assert!(coordinator.running_and_idle());
}
#[tokio::test(start_paused = true)]
async fn rearm_is_notified_when_an_armed_registration_changes() {
let coordinator = InvocationCoordinator::new();
let flag = Arc::new(AtomicBool::new(false));
let unarmed = coordinator
.register_wait(
"op",
None,
&CheckpointStatus::Started,
Some(flag),
Waker::noop(),
)
.expect("running");
drop(unarmed);
let quiet = tokio::time::timeout(Duration::from_millis(10), coordinator.rearmed()).await;
assert!(
quiet.is_err(),
"neither the registration nor its retirement notifies"
);
let armed = coordinator
.register_test_wait("op", Waker::noop())
.expect("running");
coordinator.rearmed().await;
drop(armed);
coordinator.rearmed().await;
}
#[test]
fn wake_log_records_woken_registrations_in_key_order() {
let coordinator = InvocationCoordinator::new();
let _b = coordinator
.register_test_wait("b", Waker::noop())
.expect("running");
let _a = coordinator
.register_test_wait("a", Waker::noop())
.expect("running");
coordinator.wake_operations(&["b".to_owned(), "a".to_owned(), "c".to_owned()]);
assert_eq!(coordinator.wake_log(), vec!["a".to_owned(), "b".to_owned()]);
}
}