use std::collections::BTreeMap;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use aws_sdk_lambda::types::Event;
use serde::de::DeserializeOwned;
use super::{
BackendState, CallbackFailure, CallbackReply, CallbackReplyKind, CloudApi, CloudRunner,
InvocationType, RunnerKind, TestOperation, TestResult, TestRunnerError, aws_error,
check_heartbeat, fetch_history, fold_history, is_terminal_status, reply_live, snapshot_of,
terminal_execution_event,
};
use crate::OperationStatus;
#[cfg(doc)]
use super::LocalRunner;
pub(super) const WAIT_FOR_DATA_REQUEST_RESPONSE_UNSUPPORTED: &str =
"wait_for_data with InvocationType::RequestResponse (use InvocationType::Event)";
pub(super) const SEND_CALLBACK_REQUEST_RESPONSE_UNSUPPORTED: &str =
"send_callback with InvocationType::RequestResponse (use InvocationType::Event)";
pub(super) const SEND_CALLBACK_HEARTBEAT_REQUEST_RESPONSE_UNSUPPORTED: &str =
"send_callback_heartbeat with InvocationType::RequestResponse (use InvocationType::Event)";
pub(super) const CLOUD_TIMEOUT_REPLY_UNSUPPORTED: &str =
"CallbackReply::timeout and heartbeat_timeout under CloudRunner";
pub(super) const CLOUD_PAUSE_UNSUPPORTED: &str = "pause and resume under CloudRunner";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
#[non_exhaustive]
pub enum WaitingStatus {
#[default]
Started,
Submitted,
Completed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum Selector {
Name(String, usize),
Index(usize),
Id(String),
}
impl Selector {
pub(super) fn resolve<'a>(&self, ops: &'a [TestOperation]) -> Option<&'a TestOperation> {
match self {
Self::Name(name, index) => ops
.iter()
.filter(|op| op.name.as_deref() == Some(name.as_str()))
.nth(*index),
Self::Index(index) => ops.get(*index),
Self::Id(id) => ops.iter().find(|op| op.id == *id),
}
}
}
pub(super) fn is_wait_for_callback(op: &TestOperation) -> bool {
op.kind == RunnerKind::Context && op.sub_type.as_deref() == Some(crate::callback::WFCB_SUB_TYPE)
}
pub(super) fn status_reached(
op: &TestOperation,
status: WaitingStatus,
ops: &[TestOperation],
) -> bool {
if !is_wait_for_callback(op) {
return status == WaitingStatus::Started || is_terminal_status(op.status);
}
let mut children = ops.iter().filter(|child| {
child.parent_id.as_deref() == Some(op.id.as_str())
&& matches!(child.kind, RunnerKind::Step | RunnerKind::Callback)
});
match status {
WaitingStatus::Started => children.next().is_some(),
WaitingStatus::Submitted => children.any(|child| is_terminal_status(child.status)),
WaitingStatus::Completed => {
is_terminal_status(op.status)
&& children.any(|child| {
child.kind == RunnerKind::Callback && is_terminal_status(child.status)
})
}
}
}
pub(super) fn reached_operation(
selector: &Selector,
status: WaitingStatus,
ops: &[TestOperation],
) -> Option<TestOperation> {
selector
.resolve(ops)
.filter(|op| status_reached(op, status, ops))
.cloned()
}
pub(super) fn callback_target<'a>(
selector: &Selector,
ops: &'a [TestOperation],
) -> Result<&'a TestOperation, TestRunnerError> {
let op = selector
.resolve(ops)
.ok_or(TestRunnerError::OperationNotFound)?;
match op.kind {
RunnerKind::Callback => Ok(op),
_ if is_wait_for_callback(op) => ops
.iter()
.find(|child| {
child.kind == RunnerKind::Callback
&& child.parent_id.as_deref() == Some(op.id.as_str())
})
.ok_or(TestRunnerError::CallbackNotPending),
_ => Err(TestRunnerError::NotACallback),
}
}
#[derive(Default)]
pub(super) enum RunEnd {
#[default]
Running,
Finished(Option<Box<dyn std::any::Any + Send>>),
}
impl std::fmt::Debug for RunEnd {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Running => f.write_str("Running"),
Self::Finished(_) => f.write_str("Finished"),
}
}
}
#[derive(Debug)]
pub(super) enum ControlKind {
Run,
Data {
selector: Selector,
status: WaitingStatus,
},
Pause,
Cloud,
}
impl ControlKind {
fn drives_local(&self) -> bool {
!matches!(self, Self::Cloud)
}
}
#[derive(Debug)]
pub(super) enum ControlValue {
Data(Box<TestOperation>),
PauseDone,
}
#[derive(Debug)]
pub(super) enum ControlPhase {
Dormant,
Pending {
waker: std::task::Waker,
drives_local: bool,
},
Ready(ControlValue),
InFlight {
request: u64,
waker: std::task::Waker,
},
}
#[derive(Debug)]
pub(super) struct Control {
pub(super) kind: ControlKind,
pub(super) phase: ControlPhase,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum LocalPhase {
Idle,
Invoking { tokenless_requested: bool },
Closed,
}
#[derive(Debug)]
pub(super) struct LocalState {
pub(super) records: BackendState,
pub(super) phase: LocalPhase,
pub(super) paused: bool,
pub(super) invocation_due: bool,
}
#[derive(Debug, Default)]
pub(super) struct CloudState {
pub(super) limit: Option<usize>,
pub(super) used: usize,
pub(super) execution_arn: Option<String>,
pub(super) fold: Option<Vec<TestOperation>>,
pub(super) fold_request: u64,
pub(super) next_request: u64,
}
#[derive(Debug)]
pub(super) struct RunState {
pub(super) next_id: u64,
pub(super) failure: Option<TestRunnerError>,
pub(super) end: RunEnd,
pub(super) controls: BTreeMap<u64, Control>,
pub(super) local: LocalState,
pub(super) cloud: CloudState,
#[cfg(test)]
pub(super) fail_at_commit: Option<TestRunnerError>,
}
impl RunState {
pub(super) fn new(records: BackendState) -> Self {
Self {
next_id: 0,
failure: None,
end: RunEnd::Running,
controls: BTreeMap::new(),
local: LocalState {
records,
phase: LocalPhase::Idle,
paused: false,
invocation_due: false,
},
cloud: CloudState::default(),
#[cfg(test)]
fail_at_commit: None,
}
}
pub(super) fn add_control(&mut self, kind: ControlKind) -> u64 {
self.next_id += 1;
self.controls.insert(
self.next_id,
Control {
kind,
phase: ControlPhase::Dormant,
},
);
self.next_id
}
pub(super) fn hold_open(&self) -> bool {
self.controls.values().any(|control| {
matches!(
control.phase,
ControlPhase::Pending {
drives_local: true,
..
}
)
})
}
pub(super) fn register(&mut self, id: u64, waker: &std::task::Waker) -> Option<bool> {
if self.failure.is_some() {
return None;
}
let was_open = self.hold_open();
let control = self.controls.get_mut(&id)?;
if matches!(control.phase, ControlPhase::Ready(_)) {
return None;
}
control.phase = ControlPhase::Pending {
waker: waker.clone(),
drives_local: control.kind.drives_local(),
};
Some(!was_open && self.hold_open())
}
pub(super) fn wakers(&self) -> Vec<std::task::Waker> {
self.controls
.values()
.filter_map(|control| match &control.phase {
ControlPhase::Pending { waker, .. } | ControlPhase::InFlight { waker, .. } => {
Some(waker.clone())
}
ControlPhase::Dormant | ControlPhase::Ready(_) => None,
})
.collect()
}
pub(super) fn record_failure(
&mut self,
err: TestRunnerError,
) -> (TestRunnerError, Vec<std::task::Waker>) {
let winner = self.failure.get_or_insert(err).clone();
(winner, self.wakers())
}
pub(super) fn commit<T>(
&mut self,
candidate: Result<T, TestRunnerError>,
) -> Result<T, TestRunnerError> {
#[cfg(test)]
if let Some(err) = self.fail_at_commit.take() {
self.record_failure(err);
}
match &self.failure {
Some(err) => Err(err.clone()),
None => candidate,
}
}
pub(super) fn run_in_progress(&self, name: &'static str) -> Result<(), TestRunnerError> {
match self.end {
RunEnd::Running => Ok(()),
RunEnd::Finished(_) => Err(TestRunnerError::RunFinished(name)),
}
}
pub(super) fn take_ready_data(&mut self, id: u64) -> Option<TestOperation> {
let control = self.controls.get_mut(&id)?;
match std::mem::replace(&mut control.phase, ControlPhase::Dormant) {
ControlPhase::Ready(ControlValue::Data(op)) => Some(*op),
other => {
control.phase = other;
None
}
}
}
pub(super) fn pause_done(&self, id: u64) -> bool {
self.controls.get(&id).is_some_and(|control| {
matches!(control.phase, ControlPhase::Ready(ControlValue::PauseDone))
})
}
pub(super) fn recheck(&mut self) -> Vec<std::task::Waker> {
let ops = snapshot_of(&self.local.records);
let mut wakers = Vec::new();
for control in self.controls.values_mut() {
let ControlKind::Data { selector, status } = &control.kind else {
continue;
};
if matches!(control.phase, ControlPhase::Ready(_)) {
continue;
}
let Some(op) = reached_operation(selector, *status, &ops) else {
continue;
};
let phase = std::mem::replace(
&mut control.phase,
ControlPhase::Ready(ControlValue::Data(Box::new(op))),
);
if let ControlPhase::Pending { waker, .. } = phase {
wakers.push(waker);
}
}
wakers
}
pub(super) fn admit_invocation(&mut self) -> bool {
let admitted =
self.hold_open() && !self.local.paused && self.local.phase == LocalPhase::Idle;
if admitted {
self.local.phase = LocalPhase::Invoking {
tokenless_requested: false,
};
}
admitted
}
pub(super) fn take_tokenless(&mut self) -> bool {
match &mut self.local.phase {
LocalPhase::Invoking {
tokenless_requested,
} => std::mem::take(tokenless_requested),
LocalPhase::Idle | LocalPhase::Closed => false,
}
}
pub(super) fn end_invocation(&mut self, closing: bool) -> Vec<std::task::Waker> {
if closing || self.local.phase == LocalPhase::Closed {
self.local.phase = LocalPhase::Closed;
} else {
self.local.phase = LocalPhase::Idle;
}
let mut wakers = Vec::new();
for control in self.controls.values_mut() {
if !matches!(control.kind, ControlKind::Pause) {
continue;
}
let phase = std::mem::replace(
&mut control.phase,
ControlPhase::Ready(ControlValue::PauseDone),
);
if let ControlPhase::Pending { waker, .. } = phase {
wakers.push(waker);
}
}
wakers
}
pub(super) fn settle_in_flight(&mut self, id: u64, request: u64) {
if let Some(control) = self.controls.get_mut(&id)
&& matches!(control.phase, ControlPhase::InFlight { request: r, .. } if r == request)
{
control.phase = ControlPhase::Dormant;
}
}
pub(super) fn take_poll(&mut self) -> bool {
if self.budget_spent() {
return false;
}
self.cloud.used += 1;
true
}
pub(super) fn budget_spent(&self) -> bool {
self.cloud
.limit
.is_some_and(|limit| self.cloud.used >= limit)
}
}
#[derive(Debug)]
pub(super) struct LiveState {
inner: Mutex<RunState>,
pub(super) notify: tokio::sync::Notify,
}
impl LiveState {
pub(super) fn new(records: BackendState) -> Self {
Self {
inner: Mutex::new(RunState::new(records)),
notify: tokio::sync::Notify::new(),
}
}
pub(super) fn lock(&self) -> std::sync::MutexGuard<'_, RunState> {
self.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
pub(super) async fn wait_until(&self, mut ready: impl FnMut(&mut RunState) -> bool) {
loop {
let notified = self.notify.notified();
let mut notified = std::pin::pin!(notified);
notified.as_mut().enable();
if ready(&mut self.lock()) {
return;
}
notified.await;
}
}
pub(super) fn wake_all(&self) {
let wakers = self.lock().wakers();
wake(wakers);
}
}
pub(super) fn wake(wakers: Vec<std::task::Waker>) {
for waker in wakers {
waker.wake();
}
}
pub(super) type DriverFuture =
Pin<Box<dyn Future<Output = Result<Box<dyn std::any::Any + Send>, TestRunnerError>> + Send>>;
pub(super) enum DriverSlot {
Idle(DriverFuture),
Polling,
Done,
}
pub(super) struct DriverWaker(Arc<LiveState>);
impl std::task::Wake for DriverWaker {
fn wake(self: Arc<Self>) {
self.0.wake_all();
}
fn wake_by_ref(self: &Arc<Self>) {
self.0.wake_all();
}
}
pub(super) enum Engine {
Local,
Cloud(Arc<dyn CloudControl>),
}
pub(super) struct RunCore {
pub(super) live: Arc<LiveState>,
pub(super) driver: Mutex<DriverSlot>,
pub(super) engine: Engine,
}
impl std::fmt::Debug for RunCore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RunCore")
.field("live", &self.live)
.finish_non_exhaustive()
}
}
impl RunCore {
pub(super) fn new(live: Arc<LiveState>, engine: Engine, driver: DriverFuture) -> Arc<Self> {
Arc::new(Self {
live,
driver: Mutex::new(DriverSlot::Idle(driver)),
engine,
})
}
pub(super) fn failed(live: Arc<LiveState>, engine: Engine, err: TestRunnerError) -> Arc<Self> {
live.lock().failure = Some(err);
Arc::new(Self {
live,
driver: Mutex::new(DriverSlot::Done),
engine,
})
}
pub(super) fn drive(&self, id: u64, waker: &std::task::Waker) {
let Some(opened) = self.live.lock().register(id, waker) else {
return;
};
if opened {
self.live.notify.notify_waiters();
}
self.poll_driver();
}
pub(super) fn poll_driver(&self) {
let mut driver = {
let mut slot = self.lock_driver();
match std::mem::replace(&mut *slot, DriverSlot::Polling) {
DriverSlot::Idle(driver) => driver,
other => {
*slot = other;
return;
}
}
};
let waker = std::task::Waker::from(Arc::new(DriverWaker(Arc::clone(&self.live))));
let poll = driver
.as_mut()
.poll(&mut std::task::Context::from_waker(&waker));
let std::task::Poll::Ready(outcome) = poll else {
*self.lock_driver() = DriverSlot::Idle(driver);
return;
};
let wakers = {
let mut state = self.live.lock();
match outcome {
Ok(result) => {
state.end = RunEnd::Finished(Some(result));
state.wakers()
}
Err(err) => state.record_failure(err).1,
}
};
*self.lock_driver() = DriverSlot::Done;
drop(driver);
wake(wakers);
}
pub(super) fn lock_driver(&self) -> std::sync::MutexGuard<'_, DriverSlot> {
self.driver
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
pub(super) fn leave(&self, id: u64) {
self.live.lock().controls.remove(&id);
}
#[cfg(test)]
pub(super) fn terminal_error(&self) -> Option<TestRunnerError> {
self.live.lock().failure.clone()
}
pub(super) fn late_wait(
&self,
selector: &Selector,
status: WaitingStatus,
) -> Option<Result<TestOperation, TestRunnerError>> {
let state = self.live.lock();
let err = state.failure.clone()?;
let ops = match &self.engine {
Engine::Local => snapshot_of(&state.local.records),
Engine::Cloud(_) => state.cloud.fold.clone().unwrap_or_default(),
};
Some(reached_operation(selector, status, &ops).ok_or(err))
}
pub(super) fn ended(&self) -> bool {
let state = self.live.lock();
state.failure.is_some() || !matches!(state.end, RunEnd::Running)
}
pub(super) fn commit<T>(
&self,
candidate: Result<T, TestRunnerError>,
) -> Result<T, TestRunnerError> {
self.live.lock().commit(candidate)
}
pub(super) fn pause_precondition(
&self,
state: &mut RunState,
name: &'static str,
) -> Result<(), TestRunnerError> {
if matches!(self.engine, Engine::Cloud(_)) {
return Err(TestRunnerError::Unsupported(CLOUD_PAUSE_UNSUPPORTED));
}
let precondition = state.run_in_progress(name);
state.commit(precondition)
}
pub(super) fn operations(&self) -> Option<Vec<TestOperation>> {
let state = self.live.lock();
match &self.engine {
Engine::Local => Some(snapshot_of(&state.local.records)),
Engine::Cloud(_) => state.cloud.fold.clone(),
}
}
}
pub(super) struct Registration<'a> {
pub(super) core: &'a RunCore,
pub(super) id: u64,
}
impl<'a> Registration<'a> {
pub(super) fn new(core: &'a RunCore, kind: ControlKind) -> Self {
Self {
id: core.live.lock().add_control(kind),
core,
}
}
}
impl Drop for Registration<'_> {
fn drop(&mut self) {
self.core.leave(self.id);
}
}
pub(super) async fn drive_until<T>(
registration: &Registration<'_>,
mut done: impl FnMut(&mut std::task::Context<'_>) -> Option<T>,
) -> T {
let core = registration.core;
std::future::poll_fn(|cx| {
if let Some(value) = done(cx) {
return std::task::Poll::Ready(value);
}
core.drive(registration.id, cx.waker());
done(cx).map_or(std::task::Poll::Pending, std::task::Poll::Ready)
})
.await
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum ControlRole {
Wait,
Callback,
}
pub(super) async fn await_in_flight<T>(
registration: &Registration<'_>,
role: ControlRole,
request: u64,
call: impl Future<Output = T>,
) -> Result<T, TestRunnerError> {
let core = registration.core;
let mut call = std::pin::pin!(call);
std::future::poll_fn(|cx| {
if role == ControlRole::Wait
&& let Some(err) = &core.live.lock().failure
{
return std::task::Poll::Ready(Err(err.clone()));
}
if let std::task::Poll::Ready(output) = call.as_mut().poll(cx) {
return std::task::Poll::Ready(Ok(output));
}
{
let mut state = core.live.lock();
if role == ControlRole::Wait
&& let Some(err) = &state.failure
{
return std::task::Poll::Ready(Err(err.clone()));
}
if let Some(control) = state.controls.get_mut(®istration.id) {
control.phase = ControlPhase::InFlight {
request,
waker: cx.waker().clone(),
};
}
}
core.poll_driver();
std::task::Poll::Pending
})
.await
}
#[must_use = "a RunHandle does nothing unless awaited"]
#[non_exhaustive]
pub struct RunHandle<O> {
pub(super) shared: Arc<RunCore>,
pub(super) id: u64,
pub(super) output: std::marker::PhantomData<fn() -> O>,
}
impl<O> std::fmt::Debug for RunHandle<O> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RunHandle")
.field("shared", &self.shared)
.finish_non_exhaustive()
}
}
impl<O> RunHandle<O> {
pub(super) fn from_shared(shared: Arc<RunCore>) -> Self {
let id = shared.live.lock().add_control(ControlKind::Run);
Self {
id,
shared,
output: std::marker::PhantomData,
}
}
#[must_use]
pub fn operation(&self, name: &str) -> OperationHandle {
self.handle(Selector::Name(name.to_owned(), 0))
}
#[must_use]
pub fn operation_by_index(&self, index: usize) -> OperationHandle {
self.handle(Selector::Index(index))
}
#[must_use]
pub fn operation_by_name_and_index(&self, name: &str, index: usize) -> OperationHandle {
self.handle(Selector::Name(name.to_owned(), index))
}
#[must_use]
pub fn operation_by_id(&self, id: &str) -> OperationHandle {
self.handle(Selector::Id(id.to_owned()))
}
pub(super) fn handle(&self, selector: Selector) -> OperationHandle {
OperationHandle {
shared: Arc::clone(&self.shared),
selector,
}
}
pub async fn pause(&self) -> Result<(), TestRunnerError> {
let shared = &*self.shared;
let registration = Registration::new(shared, ControlKind::Pause);
let id = registration.id;
{
let mut state = shared.live.lock();
shared.pause_precondition(&mut state, "pause")?;
state.local.paused = true;
match &mut state.local.phase {
LocalPhase::Invoking {
tokenless_requested,
} => *tokenless_requested = true,
LocalPhase::Idle | LocalPhase::Closed => return Ok(()),
}
}
drive_until(®istration, |_| {
let mut state = shared.live.lock();
(state.pause_done(id) || state.failure.is_some()).then(|| state.commit(Ok(())))
})
.await
}
#[expect(clippy::unused_async, clippy::unused_async_trait_impl)]
pub async fn resume(&self) -> Result<(), TestRunnerError> {
let shared = &*self.shared;
{
let mut state = shared.live.lock();
shared.pause_precondition(&mut state, "resume")?;
state.local.paused = false;
}
shared.live.notify.notify_waiters();
Ok(())
}
pub(super) fn take_end(&self) -> Option<Result<TestResult<O>, TestRunnerError>>
where
O: Send + 'static,
{
let mut state = self.shared.live.lock();
if let Some(err) = &state.failure {
return Some(Err(err.clone()));
}
match &mut state.end {
RunEnd::Running => None,
RunEnd::Finished(result) => result.take().map(|result| {
#[expect(clippy::expect_used)]
let result = result
.downcast::<TestResult<O>>()
.expect("the driver of a RunHandle<O> returns a TestResult<O>");
Ok(*result)
}),
}
}
}
impl<O: DeserializeOwned + Send + 'static> Future for RunHandle<O> {
type Output = Result<TestResult<O>, TestRunnerError>;
fn poll(
self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Self::Output> {
let this = self.get_mut();
if let Some(end) = this.take_end() {
this.shared.leave(this.id);
return std::task::Poll::Ready(end);
}
this.shared.drive(this.id, cx.waker());
match this.take_end() {
Some(end) => {
this.shared.leave(this.id);
std::task::Poll::Ready(end)
}
None => std::task::Poll::Pending,
}
}
}
impl<O> Drop for RunHandle<O> {
fn drop(&mut self) {
self.shared.leave(self.id);
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct OperationHandle {
pub(super) shared: Arc<RunCore>,
pub(super) selector: Selector,
}
impl OperationHandle {
pub fn snapshot(&self) -> Option<TestOperation> {
self.shared
.operations()
.and_then(|ops| self.selector.resolve(&ops).cloned())
}
pub async fn wait_for_data(
&self,
status: WaitingStatus,
) -> Result<TestOperation, TestRunnerError> {
let shared = &*self.shared;
if let Engine::Cloud(cloud) = &shared.engine {
return cloud
.wait_for_data(&self.shared, &self.selector, status)
.await;
}
if let Some(late) = shared.late_wait(&self.selector, status) {
return late;
}
let registration = Registration::new(
shared,
ControlKind::Data {
selector: self.selector.clone(),
status,
},
);
let id = registration.id;
drive_until(®istration, |_| {
let mut state = shared.live.lock();
let ready = state.take_ready_data(id).or_else(|| {
reached_operation(&self.selector, status, &snapshot_of(&state.local.records))
});
let candidate = match ready {
Some(op) => Ok(op),
None if state.failure.is_some() || matches!(state.end, RunEnd::Finished(_)) => {
Err(TestRunnerError::OperationNotReached(status))
}
None => return None,
};
Some(state.commit(candidate))
})
.await
}
pub async fn send_callback(&self, reply: CallbackReply) -> Result<(), TestRunnerError> {
let shared = &*self.shared;
if let Engine::Cloud(cloud) = &shared.engine {
return cloud
.send_callback(&self.shared, &self.selector, reply)
.await;
}
let wakers = reply_live(&mut shared.live.lock(), &self.selector, reply)?;
wake(wakers);
Ok(())
}
pub async fn send_callback_heartbeat(&self) -> Result<(), TestRunnerError> {
let shared = &*self.shared;
if let Engine::Cloud(cloud) = &shared.engine {
return cloud.send_heartbeat(&self.shared, &self.selector).await;
}
check_heartbeat(&mut shared.live.lock(), &self.selector)
}
}
pub(super) type ControlFuture<'a, T> =
Pin<Box<dyn Future<Output = Result<T, TestRunnerError>> + Send + 'a>>;
pub(super) trait CloudControl: Send + Sync {
fn wait_for_data<'a>(
&'a self,
core: &'a RunCore,
selector: &'a Selector,
status: WaitingStatus,
) -> ControlFuture<'a, TestOperation>;
fn send_callback<'a>(
&'a self,
core: &'a RunCore,
selector: &'a Selector,
reply: CallbackReply,
) -> ControlFuture<'a, ()>;
fn send_heartbeat<'a>(
&'a self,
core: &'a RunCore,
selector: &'a Selector,
) -> ControlFuture<'a, ()>;
}
#[derive(Debug)]
pub(super) enum HistoryStart {
Read(u64),
Done(Option<Vec<TestOperation>>),
}
pub(super) struct CloudShared<A> {
pub(super) invocation_type: InvocationType,
pub(super) poll_interval: std::time::Duration,
pub(super) api: std::sync::OnceLock<Arc<A>>,
pub(super) live: Arc<LiveState>,
}
impl<A> CloudShared<A> {
pub(super) fn new(runner: &CloudRunner, live: Arc<LiveState>) -> Self {
live.lock().cloud.limit = runner.max_poll_attempts;
Self {
invocation_type: runner.invocation_type,
poll_interval: runner.poll_interval,
api: std::sync::OnceLock::new(),
live,
}
}
pub(super) fn set_target(&self, api: Arc<A>, execution_arn: String) {
let _ = self.api.set(api);
self.live.lock().cloud.execution_arn = Some(execution_arn);
}
pub(super) fn take_poll(&self) -> bool {
self.live.lock().take_poll()
}
pub(super) fn budget_spent(&self) -> bool {
self.live.lock().budget_spent()
}
pub(super) fn used(&self) -> usize {
self.live.lock().cloud.used
}
pub(super) fn begin_history(&self, role: ControlRole) -> Result<HistoryStart, TestRunnerError> {
let mut state = self.live.lock();
if role == ControlRole::Wait {
if let Some(err) = &state.failure {
return Err(err.clone());
}
if matches!(state.end, RunEnd::Finished(_)) {
return Ok(HistoryStart::Done(state.cloud.fold.clone()));
}
}
if !state.take_poll() {
return Ok(HistoryStart::Done(state.cloud.fold.clone()));
}
state.cloud.next_request += 1;
Ok(HistoryStart::Read(state.cloud.next_request))
}
pub(super) fn finish_history(
&self,
reader: Option<(u64, ControlRole)>,
request: u64,
fetched: Result<Vec<Event>, TestRunnerError>,
) -> Result<(Vec<Event>, Vec<TestOperation>), TestRunnerError> {
let mut state = self.live.lock();
if let Some((id, role)) = reader {
state.settle_in_flight(id, request);
if role == ControlRole::Wait
&& let Some(err) = &state.failure
{
return Err(err.clone());
}
}
let events = fetched?;
let ops = fold_history(&events);
if request > state.cloud.fold_request {
state.cloud.fold = Some(ops.clone());
state.cloud.fold_request = request;
}
Ok((events, ops))
}
pub(super) fn begin_request(&self) -> u64 {
let mut state = self.live.lock();
state.cloud.next_request += 1;
state.cloud.next_request
}
pub(super) fn finish_call(
&self,
id: u64,
request: u64,
sent: Result<(), TestRunnerError>,
) -> Result<(), TestRunnerError> {
self.live.lock().settle_in_flight(id, request);
sent
}
}
impl<A: CloudApi + Send + Sync + 'static> CloudShared<A> {
pub(super) async fn until_target(
&self,
registration: &Registration<'_>,
) -> Result<(Arc<A>, String), TestRunnerError> {
drive_until(registration, |_| {
let state = self.live.lock();
if let Some(arn) = &state.cloud.execution_arn
&& let Some(api) = self.api.get()
{
return Some(Ok((Arc::clone(api), arn.clone())));
}
state.failure.clone().map(Err)
})
.await
}
pub(super) async fn driver_history(
&self,
api: &A,
execution_arn: &str,
) -> Result<(Vec<Event>, Vec<TestOperation>), TestRunnerError> {
let request = self.begin_request();
let fetched = fetch_history(api, execution_arn, self.poll_interval).await;
self.finish_history(None, request, fetched)
}
pub(super) async fn control_operations(
&self,
registration: &Registration<'_>,
) -> Result<(Arc<A>, Vec<TestOperation>), TestRunnerError> {
let (api, execution_arn) = self.until_target(registration).await?;
match self.begin_history(ControlRole::Callback)? {
HistoryStart::Read(request) => {
let fetched = await_in_flight(
registration,
ControlRole::Callback,
request,
fetch_history(&*api, &execution_arn, self.poll_interval),
)
.await?;
let (_, ops) = self.finish_history(
Some((registration.id, ControlRole::Callback)),
request,
fetched,
)?;
Ok((api, ops))
}
HistoryStart::Done(fold) => Ok((api, fold.ok_or(TestRunnerError::OperationNotFound)?)),
}
}
pub(super) async fn drive_for(registration: &Registration<'_>, duration: std::time::Duration) {
let core = registration.core;
let mut sleep = std::pin::pin!(tokio::time::sleep(duration));
drive_until(registration, |cx| {
(sleep.as_mut().poll(cx).is_ready() || core.ended()).then_some(())
})
.await;
}
async fn wait_for_data_outcome(
&self,
core: &RunCore,
selector: &Selector,
status: WaitingStatus,
) -> Result<TestOperation, TestRunnerError> {
let registration = Registration::new(core, ControlKind::Cloud);
let (api, execution_arn) = self.until_target(®istration).await?;
loop {
Self::drive_for(®istration, self.poll_interval).await;
let request = match self.begin_history(ControlRole::Wait)? {
HistoryStart::Read(request) => request,
HistoryStart::Done(fold) => {
return reached_operation(selector, status, &fold.unwrap_or_default())
.ok_or(TestRunnerError::OperationNotReached(status));
}
};
let fetched = await_in_flight(
®istration,
ControlRole::Wait,
request,
fetch_history(&*api, &execution_arn, self.poll_interval),
)
.await?;
let (events, ops) =
self.finish_history(Some((registration.id, ControlRole::Wait)), request, fetched)?;
if let Some(op) = reached_operation(selector, status, &ops) {
return Ok(op);
}
if terminal_execution_event(&events).is_some() {
return Err(TestRunnerError::OperationNotReached(status));
}
}
}
async fn send_callback_outcome(
&self,
core: &RunCore,
selector: &Selector,
reply: CallbackReply,
) -> Result<(), TestRunnerError> {
if self.invocation_type == InvocationType::RequestResponse {
return Err(TestRunnerError::Unsupported(
SEND_CALLBACK_REQUEST_RESPONSE_UNSUPPORTED,
));
}
let call = match reply.kind {
CallbackReplyKind::Success(payload) => CallbackCall::Success(payload),
CallbackReplyKind::Failure(failure) => CallbackCall::Failure(failure),
CallbackReplyKind::Timeout | CallbackReplyKind::HeartbeatTimeout => {
return Err(TestRunnerError::Unsupported(
CLOUD_TIMEOUT_REPLY_UNSUPPORTED,
));
}
};
let registration = Registration::new(core, ControlKind::Cloud);
let (api, ops) = self.control_operations(®istration).await?;
let target = callback_target(selector, &ops)?;
if target.status != OperationStatus::Started {
return Err(TestRunnerError::CallbackNotPending);
}
let id = callback_id(target)?;
let request = self.begin_request();
let sent = match call {
CallbackCall::Success(payload) => {
await_in_flight(
®istration,
ControlRole::Callback,
request,
api.send_callback_success(&id, payload.into_bytes()),
)
.await?
}
CallbackCall::Failure(failure) => {
await_in_flight(
®istration,
ControlRole::Callback,
request,
api.send_callback_failure(&id, failure.error_object()),
)
.await?
}
};
self.finish_call(registration.id, request, sent.map_err(aws_error))
}
async fn send_heartbeat_outcome(
&self,
core: &RunCore,
selector: &Selector,
) -> Result<(), TestRunnerError> {
if self.invocation_type == InvocationType::RequestResponse {
return Err(TestRunnerError::Unsupported(
SEND_CALLBACK_HEARTBEAT_REQUEST_RESPONSE_UNSUPPORTED,
));
}
let registration = Registration::new(core, ControlKind::Cloud);
let (api, ops) = self.control_operations(®istration).await?;
let id = callback_id(callback_target(selector, &ops)?)?;
let request = self.begin_request();
let sent = await_in_flight(
®istration,
ControlRole::Callback,
request,
api.send_callback_heartbeat(&id),
)
.await?;
self.finish_call(registration.id, request, sent.map_err(aws_error))
}
}
pub(super) fn callback_id(op: &TestOperation) -> Result<String, TestRunnerError> {
op.callback
.callback_id
.clone()
.ok_or(TestRunnerError::CallbackNotPending)
}
impl<A: CloudApi + Send + Sync + 'static> CloudControl for CloudShared<A> {
fn wait_for_data<'a>(
&'a self,
core: &'a RunCore,
selector: &'a Selector,
status: WaitingStatus,
) -> ControlFuture<'a, TestOperation> {
Box::pin(async move {
if self.invocation_type == InvocationType::RequestResponse {
return Err(TestRunnerError::Unsupported(
WAIT_FOR_DATA_REQUEST_RESPONSE_UNSUPPORTED,
));
}
if let Some(late) = core.late_wait(selector, status) {
return late;
}
core.commit(self.wait_for_data_outcome(core, selector, status).await)
})
}
fn send_callback<'a>(
&'a self,
core: &'a RunCore,
selector: &'a Selector,
reply: CallbackReply,
) -> ControlFuture<'a, ()> {
Box::pin(self.send_callback_outcome(core, selector, reply))
}
fn send_heartbeat<'a>(
&'a self,
core: &'a RunCore,
selector: &'a Selector,
) -> ControlFuture<'a, ()> {
Box::pin(self.send_heartbeat_outcome(core, selector))
}
}
pub(super) enum CallbackCall {
Success(String),
Failure(CallbackFailure),
}