1use std::future::Future;
9use std::sync::Arc;
10use std::time::Duration;
11
12use serde::{Serialize, de::DeserializeOwned};
13
14use crate::BoxError;
15use crate::builders::{
16 ChildBuilder, CreateCallbackBuilder, InvokeBuilder, JoinAllBuilder, MapBuilder,
17 ParallelBuilder, ParallelMixedBuilder, RaceBuilder, SelectOkBuilder, StepBuilder, Submitter,
18 TryJoinAllBuilder, WaitBuilder, WaitForCallbackBuilder, WaitForConditionBuilder,
19 WithRetryBuilder,
20};
21use crate::checkpoint_coalescer::{
22 Admission, BatchLimits, CheckpointBatch, CheckpointCoalescer, TrackedUpdate,
23 split_into_requests,
24};
25use crate::client::{
26 CheckpointOutput, ExecutionClient, ExecutionClientError, ExecutionClientErrorScope,
27};
28use crate::coordinator::{InvocationCoordinator, PermitKind};
29use crate::driver::{FatalError, FatalSeverity, TaskOwnership};
30use crate::engine::{
31 CheckpointLog, CheckpointRecord, CheckpointStatus, CheckpointStatusView, EngineState,
32 OperationId, TerminalReplaySnapshot,
33};
34use crate::error::{
35 NonDeterministicExecutionError, NonDeterministicExecutionErrorKind, OperationError,
36 OperationErrorKind, StepError, StepErrorKind,
37};
38use crate::future::{Branch, DurableFuture};
39
40use aws_sdk_lambda::types::{OperationAction, OperationUpdate};
41use tokio::sync::Mutex;
42
43#[derive(Default)]
45struct ChangeSet {
46 ids: Vec<String>,
49 executions: Vec<crate::plugin::OperationInfo>,
51}
52
53pub(crate) struct SubmittedWrite {
62 result: std::pin::Pin<
63 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send>,
64 >,
65 #[cfg(test)]
67 first_poll: Arc<std::sync::atomic::AtomicBool>,
68 report: Option<Box<WriteReport>>,
72}
73
74struct WriteReport {
77 plugins: Arc<crate::plugin::InvocationPlugins>,
78 log: Arc<CheckpointLog>,
79 ready: Option<Result<CheckpointOutput, ExecutionClientError>>,
81 wait: crate::plugin::DeliveryWait,
84}
85
86impl Drop for WriteReport {
87 fn drop(&mut self) {
89 self.plugins.cancel_wait(&mut self.wait);
90 }
91}
92
93impl SubmittedWrite {
94 fn new(
95 result: impl Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + 'static,
96 ) -> Self {
97 Self {
98 result: Box::pin(result),
99 #[cfg(test)]
100 first_poll: Arc::new(std::sync::atomic::AtomicBool::new(false)),
101 report: None,
102 }
103 }
104
105 fn reported_by(mut self, ctx: &DurableContext) -> Self {
111 self.report = ctx.shared_plugins().map(|plugins| {
112 Box::new(WriteReport {
113 plugins,
114 log: Arc::clone(&ctx.inner.engine.checkpoint_log),
115 ready: None,
116 wait: crate::plugin::DeliveryWait::default(),
117 })
118 });
119 self
120 }
121
122 #[cfg(test)]
125 pub(crate) fn from_receiver(
126 receiver: tokio::sync::oneshot::Receiver<Result<CheckpointOutput, ExecutionClientError>>,
127 ) -> Self {
128 Self::new(async move {
129 match receiver.await {
130 Ok(result) => result,
131 Err(_closed) => std::future::pending().await,
132 }
133 })
134 }
135
136 fn rejected() -> Self {
139 Self::new(std::future::pending())
140 }
141}
142
143impl Future for SubmittedWrite {
144 type Output = Result<CheckpointOutput, ExecutionClientError>;
145
146 fn poll(
148 self: std::pin::Pin<&mut Self>,
149 cx: &mut std::task::Context<'_>,
150 ) -> std::task::Poll<Self::Output> {
151 let this = self.get_mut();
152 #[cfg(test)]
153 this.first_poll
154 .store(true, std::sync::atomic::Ordering::SeqCst);
155 let Some(report) = this.report.as_mut() else {
156 return this.result.as_mut().poll(cx);
157 };
158 let result = match report.ready.take() {
159 Some(result) => result,
160 None => std::task::ready!(this.result.as_mut().poll(cx)),
161 };
162 if report
163 .plugins
164 .poll_changes_delivered(&report.log, &mut report.wait, cx)
165 .is_pending()
166 {
167 report.ready = Some(result);
168 return std::task::Poll::Pending;
169 }
170 this.report = None;
171 std::task::Poll::Ready(result)
172 }
173}
174
175pub(crate) struct ReservedWrite {
190 write: SubmittedWrite,
191 hold: Option<Arc<CheckpointCoalescer>>,
194}
195
196impl ReservedWrite {
197 fn rejected() -> Self {
200 Self {
201 write: SubmittedWrite::rejected(),
202 hold: None,
203 }
204 }
205
206 pub(crate) fn release(mut self) -> SubmittedWrite {
209 std::mem::replace(&mut self.write, SubmittedWrite::rejected())
212 }
213}
214
215impl Drop for ReservedWrite {
216 fn drop(&mut self) {
219 if let Some(hold) = self.hold.take() {
220 hold.release_hold();
221 }
222 }
223}
224
225#[cfg(test)]
229pub(crate) type PreLockGate =
230 Arc<dyn Fn(&[OperationUpdate]) -> Option<Arc<tokio::sync::Notify>> + Send + Sync>;
231
232#[cfg(test)]
237pub(crate) type SubmissionRecorder =
238 Arc<dyn Fn(&[OperationUpdate], &Arc<std::sync::atomic::AtomicBool>) + Send + Sync>;
239
240#[cfg(test)]
244pub(crate) type SettlerEntryRecorder = Arc<dyn Fn(&str, OperationAction) + Send + Sync>;
245
246#[cfg(test)]
248#[derive(Default)]
249pub(crate) struct TestHooks {
250 gate: Option<PreLockGate>,
251 submissions: Option<SubmissionRecorder>,
252 settler_entries: Option<SettlerEntryRecorder>,
253}
254
255fn batch_result(batch: Arc<CheckpointBatch>) -> SubmittedWrite {
258 SubmittedWrite::new(async move {
259 loop {
260 let mut notified = std::pin::pin!(batch.notified());
264 notified.as_mut().enable();
265 if let Some(result) = batch.result_clone() {
266 return result;
267 }
268 notified.await;
269 }
270 })
271}
272
273fn checkpoint_failure_fatal(err: &ExecutionClientError) -> FatalError {
279 if err.is_retryable() {
280 FatalError {
281 error_type: crate::error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE.to_owned(),
282 error_message: crate::error::checkpoint_invocation_failure_message(err),
283 severity: FatalSeverity::FailInvocation,
284 }
285 } else {
286 let wire = crate::error::checkpoint_execution_failure_wire(err);
287 FatalError {
288 error_type: wire.error_type().unwrap_or_default().to_owned(),
289 error_message: wire.error_message().unwrap_or_default().to_owned(),
290 severity: FatalSeverity::FailExecution,
291 }
292 }
293}
294
295#[derive(Debug)]
297pub(crate) enum WriteError {
298 Failed(ExecutionClientError),
301 Stopped,
306}
307
308#[derive(Debug, Clone)]
313pub(crate) enum ActiveBody {
314 Context(String),
317 Step(String),
320 Plugin,
325 Outcome(String),
331 Unset,
335}
336
337pub(crate) fn current_body() -> ActiveBody {
339 ACTIVE_BODY
340 .try_with(ActiveBody::clone)
341 .unwrap_or(ActiveBody::Unset)
342}
343
344tokio::task_local! {
345 static ACTIVE_BODY: ActiveBody;
350}
351
352pub(crate) fn run_body<F: Future>(body: ActiveBody, fut: F) -> impl Future<Output = F::Output> {
357 ACTIVE_BODY.scope(body, fut)
358}
359
360pub(crate) fn run_body_sync<R>(body: ActiveBody, f: impl FnOnce() -> R) -> R {
368 ACTIVE_BODY.sync_scope(body, f)
369}
370
371fn remaining_until(deadline_ms: u64, now: std::time::SystemTime) -> Option<Duration> {
375 if deadline_ms == 0 {
376 return None;
377 }
378 let deadline = std::time::UNIX_EPOCH + Duration::from_millis(deadline_ms);
379 Some(deadline.duration_since(now).unwrap_or(Duration::ZERO))
380}
381
382pub(crate) fn display_prefix(prefix: &str) -> &str {
384 if prefix.is_empty() { "root" } else { prefix }
385}
386
387fn wfc_retries(updates: &[OperationUpdate]) -> Vec<(String, Option<String>)> {
390 updates
391 .iter()
392 .filter(|update| {
393 *update.action() == OperationAction::Retry
394 && update.sub_type() == Some(crate::wait_for_condition::WFC_SUB_TYPE)
395 })
396 .map(|update| (update.id().to_owned(), update.payload().map(str::to_owned)))
397 .collect()
398}
399
400fn mark_retry_pending(log: &CheckpointLog, wire_id: &str, payload: Option<&str>) {
412 log.update(wire_id, |record| {
413 if record.status != CheckpointStatus::Pending {
414 record.status = CheckpointStatus::Pending;
415 record.result = payload.map(str::to_owned);
416 }
417 });
418}
419
420struct Inner {
422 execution_arn: String,
423 lambda_context: lambda_runtime::Context,
424 engine: Arc<EngineState>,
427 coordinator: Arc<InvocationCoordinator>,
430 task_ownership: Arc<TaskOwnership>,
432 execution_client: Option<Arc<dyn ExecutionClient>>,
434 checkpoint_token: Arc<Mutex<String>>,
439 coalescer: Arc<CheckpointCoalescer>,
446 #[cfg(test)]
449 hooks: Arc<std::sync::Mutex<TestHooks>>,
450 parent_wire_id: Option<String>,
453 admission: Arc<Admission>,
456 owners: Arc<[String]>,
461 replay_span: tracing::Span,
475 plugins: Option<Arc<crate::plugin::InvocationPlugins>>,
479 nesting_level: u32,
484 preserve_child_depth: u32,
488 rebuilding: bool,
491}
492
493impl std::fmt::Debug for Inner {
494 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
498 f.debug_struct("Inner")
499 .field("execution_arn", &self.execution_arn)
500 .field("parent_wire_id", &self.parent_wire_id)
501 .field("is_replaying", &self.engine.is_replaying())
502 .field("checkpoint_token", &"<redacted>")
503 .finish_non_exhaustive()
504 }
505}
506
507#[derive(Clone)]
546#[non_exhaustive]
547pub struct DurableContext {
548 inner: Arc<Inner>,
549 awaited: Option<Arc<std::sync::atomic::AtomicBool>>,
555}
556
557impl std::fmt::Debug for DurableContext {
558 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
562 f.debug_struct("DurableContext")
563 .field("execution_arn", &self.inner.execution_arn)
564 .field("parent_wire_id", &self.inner.parent_wire_id)
565 .field("is_replaying", &self.inner.engine.is_replaying())
566 .field("checkpoint_token", &"<redacted>")
567 .finish_non_exhaustive()
568 }
569}
570
571impl DurableContext {
572 #[cfg(test)]
575 pub(crate) fn test_context() -> Self {
576 let execution_arn = String::from("arn:aws:lambda:us-east-1:123456789012:function:test");
577 let lambda_context = lambda_runtime::Context::default();
578 let engine = Arc::new(EngineState::new_root(Arc::new(CheckpointLog::empty())));
579 let replay_span = crate::tracing_layer::execution_span(
580 &execution_arn,
581 &lambda_context.request_id,
582 lambda_context.tenant_id.as_deref(),
583 engine.is_replaying(),
584 );
585 Self {
586 inner: Arc::new(Inner {
587 execution_arn,
588 lambda_context,
589 engine,
590 coordinator: InvocationCoordinator::new(),
591 task_ownership: Arc::new(TaskOwnership::new_current()),
592 execution_client: None,
593 checkpoint_token: Arc::new(Mutex::new(String::new())),
594 coalescer: Arc::new(CheckpointCoalescer::new(Duration::ZERO)),
595 #[cfg(test)]
596 hooks: Arc::default(),
597 parent_wire_id: None,
598 admission: Arc::new(Admission::new()),
599 owners: Arc::from(Vec::new()),
600 replay_span,
601 plugins: None,
602 nesting_level: 0,
603 preserve_child_depth: 0,
604 rebuilding: false,
605 }),
606 awaited: None,
607 }
608 }
609
610 #[cfg(test)]
615 pub(crate) fn new_root(
616 execution_arn: String,
617 lambda_context: lambda_runtime::Context,
618 checkpoint_log: Arc<CheckpointLog>,
619 ) -> Self {
620 let engine = Arc::new(EngineState::new_root(checkpoint_log));
621 let replay_span = crate::tracing_layer::execution_span(
622 &execution_arn,
623 &lambda_context.request_id,
624 lambda_context.tenant_id.as_deref(),
625 engine.is_replaying(),
626 );
627 Self {
628 inner: Arc::new(Inner {
629 execution_arn,
630 lambda_context,
631 engine,
632 coordinator: InvocationCoordinator::new(),
633 task_ownership: Arc::new(TaskOwnership::new_current()),
634 execution_client: None,
635 checkpoint_token: Arc::new(Mutex::new(String::new())),
636 coalescer: Arc::new(CheckpointCoalescer::new(Duration::ZERO)),
637 #[cfg(test)]
638 hooks: Arc::default(),
639 parent_wire_id: None,
640 admission: Arc::new(Admission::new()),
641 owners: Arc::from(Vec::new()),
642 replay_span,
643 plugins: None,
644 nesting_level: 0,
645 preserve_child_depth: 0,
646 rebuilding: false,
647 }),
648 awaited: None,
649 }
650 }
651
652 #[cfg(test)]
655 pub(crate) fn new_root_with_client(
656 execution_arn: String,
657 lambda_context: lambda_runtime::Context,
658 checkpoint_log: Arc<CheckpointLog>,
659 client: Arc<dyn ExecutionClient>,
660 checkpoint_token: String,
661 ) -> Self {
662 let engine = Arc::new(EngineState::new_root(checkpoint_log));
663 let replay_span = crate::tracing_layer::execution_span(
664 &execution_arn,
665 &lambda_context.request_id,
666 lambda_context.tenant_id.as_deref(),
667 engine.is_replaying(),
668 );
669 Self {
670 inner: Arc::new(Inner {
671 execution_arn,
672 lambda_context,
673 engine,
674 coordinator: InvocationCoordinator::new(),
675 task_ownership: Arc::new(TaskOwnership::new_current()),
676 execution_client: Some(client),
677 checkpoint_token: Arc::new(Mutex::new(checkpoint_token)),
678 coalescer: Arc::new(CheckpointCoalescer::new(Duration::ZERO)),
679 #[cfg(test)]
680 hooks: Arc::default(),
681 parent_wire_id: None,
682 admission: Arc::new(Admission::new()),
683 owners: Arc::from(Vec::new()),
684 replay_span,
685 plugins: None,
686 nesting_level: 0,
687 preserve_child_depth: 0,
688 rebuilding: false,
689 }),
690 awaited: None,
691 }
692 }
693
694 #[expect(clippy::too_many_arguments, reason = "one argument per root setting")]
704 pub(crate) fn new_root_with_client_and_defaults(
705 execution_arn: String,
706 lambda_context: lambda_runtime::Context,
707 checkpoint_log: Arc<CheckpointLog>,
708 client: Arc<dyn ExecutionClient>,
709 checkpoint_token: String,
710 checkpoint_buffer_window: Duration,
711 preserve_child_depth: u32,
712 plugins: Option<Arc<crate::plugin::InvocationPlugins>>,
713 ) -> Self {
714 let engine = Arc::new(EngineState::new_root(checkpoint_log));
715 let replay_span = crate::tracing_layer::execution_span(
716 &execution_arn,
717 &lambda_context.request_id,
718 lambda_context.tenant_id.as_deref(),
719 engine.is_replaying(),
720 );
721 Self {
722 inner: Arc::new(Inner {
723 execution_arn,
724 lambda_context,
725 engine,
726 coordinator: InvocationCoordinator::new(),
727 task_ownership: Arc::new(TaskOwnership::new_current()),
728 execution_client: Some(client),
729 checkpoint_token: Arc::new(Mutex::new(checkpoint_token)),
730 coalescer: Arc::new(CheckpointCoalescer::new(checkpoint_buffer_window)),
731 #[cfg(test)]
732 hooks: Arc::default(),
733 parent_wire_id: None,
734 admission: Arc::new(Admission::new()),
735 owners: Arc::from(Vec::new()),
736 replay_span,
737 plugins,
738 nesting_level: 0,
739 preserve_child_depth,
740 rebuilding: false,
741 }),
742 awaited: None,
743 }
744 }
745
746 #[cfg(test)]
751 pub(crate) fn test_context_with_depth(
752 depth: u32,
753 ) -> (Self, Arc<crate::client::InMemoryExecutionClient>) {
754 let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
755 let ctx = Self::test_context_with_depth_and_client(
756 depth,
757 Arc::clone(&client) as Arc<dyn ExecutionClient>,
758 );
759 (ctx, client)
760 }
761
762 #[cfg(test)]
764 pub(crate) fn test_context_with_depth_and_client(
765 depth: u32,
766 client: Arc<dyn ExecutionClient>,
767 ) -> Self {
768 Self::new_root_with_client_and_defaults(
769 "arn:test".to_owned(),
770 lambda_runtime::Context::default(),
771 Arc::new(CheckpointLog::empty()),
772 client,
773 "token0".to_owned(),
774 Duration::ZERO,
775 depth,
776 None,
777 )
778 }
779
780 #[cfg(test)]
784 pub(crate) fn new_root_with_client_and_coalescer(
785 execution_arn: String,
786 lambda_context: lambda_runtime::Context,
787 checkpoint_log: Arc<CheckpointLog>,
788 client: Arc<dyn ExecutionClient>,
789 checkpoint_token: String,
790 coalescer: CheckpointCoalescer,
791 ) -> Self {
792 let engine = Arc::new(EngineState::new_root(checkpoint_log));
793 let replay_span = crate::tracing_layer::execution_span(
794 &execution_arn,
795 &lambda_context.request_id,
796 lambda_context.tenant_id.as_deref(),
797 engine.is_replaying(),
798 );
799 Self {
800 inner: Arc::new(Inner {
801 execution_arn,
802 lambda_context,
803 engine,
804 coordinator: InvocationCoordinator::new(),
805 task_ownership: Arc::new(TaskOwnership::new_current()),
806 execution_client: Some(client),
807 checkpoint_token: Arc::new(Mutex::new(checkpoint_token)),
808 coalescer: Arc::new(coalescer),
809 #[cfg(test)]
810 hooks: Arc::default(),
811 parent_wire_id: None,
812 admission: Arc::new(Admission::new()),
813 owners: Arc::from(Vec::new()),
814 replay_span,
815 plugins: None,
816 nesting_level: 0,
817 preserve_child_depth: 0,
818 rebuilding: false,
819 }),
820 awaited: None,
821 }
822 }
823 pub(crate) fn new_child(&self, parent_positional_id: &str, name: Option<&str>) -> Self {
828 self.new_child_rebuilding(parent_positional_id, name, false)
829 }
830
831 pub(crate) fn new_child_rebuilding(
837 &self,
838 parent_positional_id: &str,
839 name: Option<&str>,
840 rebuilding: bool,
841 ) -> Self {
842 let wire_id = crate::engine::compute_wire_id_public(parent_positional_id);
843 let engine = Arc::new(EngineState::new_child(
844 parent_positional_id,
845 Arc::clone(&self.inner.engine.checkpoint_log),
846 ));
847 let replay_span = crate::tracing_layer::scoped_execution_span(
858 &self.inner.execution_arn,
859 &self.inner.lambda_context.request_id,
860 self.inner.lambda_context.tenant_id.as_deref(),
861 &wire_id,
862 name,
863 engine.is_replaying(),
864 );
865 Self {
866 inner: Arc::new(Inner {
867 execution_arn: self.inner.execution_arn.clone(),
868 lambda_context: self.inner.lambda_context.clone(),
869 engine,
870 coordinator: Arc::clone(&self.inner.coordinator),
871 task_ownership: Arc::clone(&self.inner.task_ownership),
872 execution_client: self.inner.execution_client.clone(),
873 checkpoint_token: Arc::clone(&self.inner.checkpoint_token),
874 coalescer: Arc::clone(&self.inner.coalescer),
875 #[cfg(test)]
876 hooks: Arc::clone(&self.inner.hooks),
877 parent_wire_id: Some(wire_id),
878 admission: Arc::clone(&self.inner.admission),
879 owners: self.owners_with(parent_positional_id),
880 replay_span,
881 plugins: self.inner.plugins.clone(),
882 nesting_level: self.inner.nesting_level.saturating_add(1),
883 preserve_child_depth: self.inner.preserve_child_depth,
884 rebuilding: self.inner.rebuilding || rebuilding,
885 }),
886 awaited: None,
887 }
888 }
889
890 pub(crate) fn new_virtual_child(&self, child_positional_id: &str, name: Option<&str>) -> Self {
900 let wire_id = crate::engine::compute_wire_id_public(child_positional_id);
901 let engine = Arc::new(EngineState::new_child(
902 child_positional_id,
903 Arc::clone(&self.inner.engine.checkpoint_log),
904 ));
905 let replay_span = crate::tracing_layer::scoped_execution_span(
910 &self.inner.execution_arn,
911 &self.inner.lambda_context.request_id,
912 self.inner.lambda_context.tenant_id.as_deref(),
913 &wire_id,
914 name,
915 engine.is_replaying(),
916 );
917 Self {
918 inner: Arc::new(Inner {
919 execution_arn: self.inner.execution_arn.clone(),
920 lambda_context: self.inner.lambda_context.clone(),
921 engine,
922 coordinator: Arc::clone(&self.inner.coordinator),
923 task_ownership: Arc::clone(&self.inner.task_ownership),
924 execution_client: self.inner.execution_client.clone(),
925 checkpoint_token: Arc::clone(&self.inner.checkpoint_token),
926 coalescer: Arc::clone(&self.inner.coalescer),
927 #[cfg(test)]
928 hooks: Arc::clone(&self.inner.hooks),
929 parent_wire_id: self.inner.parent_wire_id.clone(),
930 admission: Arc::clone(&self.inner.admission),
931 owners: Arc::clone(&self.inner.owners),
932 replay_span,
933 plugins: self.inner.plugins.clone(),
934 nesting_level: self.inner.nesting_level,
935 preserve_child_depth: self.inner.preserve_child_depth,
936 rebuilding: self.inner.rebuilding,
937 }),
938 awaited: None,
939 }
940 }
941
942 pub(crate) fn reject_record_at_virtual_child(
950 &self,
951 op_id: &OperationId,
952 sub_type: &str,
953 name: Option<&str>,
954 ) -> Result<(), OperationError> {
955 let Some(expected) = self.with_checkpoint_record(op_id.positional(), |record| {
956 format_op_identity(
957 record.op_type.as_deref().unwrap_or("Unknown"),
958 record.sub_type.as_deref(),
959 record.op_name.as_deref(),
960 )
961 }) else {
962 return Ok(());
963 };
964 Err(self.replay_mismatch_error(
965 op_id.wire(),
966 &expected,
967 "virtual Context",
968 Some(sub_type),
969 name,
970 ))
971 }
972
973 pub(crate) fn namespace_is_replaying(&self, child_positional_id: &str) -> bool {
978 EngineState::new_child(
979 child_positional_id,
980 Arc::clone(&self.inner.engine.checkpoint_log),
981 )
982 .is_replaying()
983 }
984
985 pub(crate) fn preserves_children(&self, levels_below: u32) -> bool {
988 let level = self.inner.nesting_level.saturating_add(levels_below);
989 level <= self.inner.preserve_child_depth
990 }
991
992 pub(crate) fn is_rebuilding(&self) -> bool {
995 self.inner.rebuilding
996 }
997
998 pub(crate) fn mint_id(&self) -> OperationId {
1007 let id = self.inner.engine.mint_id();
1008 self.inner.replay_span.record(
1009 crate::tracing_layer::fields::IS_REPLAY,
1010 self.inner.engine.is_replaying(),
1011 );
1012 id
1013 }
1014
1015 pub(crate) fn prefix(&self) -> &str {
1018 self.inner.engine.id_counter.prefix()
1019 }
1020
1021 pub(crate) fn claim_operation_id(&self, method: &'static str) -> OperationId {
1057 let message = ACTIVE_BODY
1058 .try_with(|active| match active {
1059 ActiveBody::Context(prefix) if prefix == self.prefix() => None,
1060 ActiveBody::Context(prefix) => Some(format!(
1061 "context usage error in \"{method}\": the operation was called on context \
1062 \"{calling}\", but the SDK is running the body of context \"{active}\". \
1063 Inside run_in_child_context, with_retry, map and parallel, call operations \
1064 on the context passed to the closure.",
1065 calling = display_prefix(self.prefix()),
1066 active = display_prefix(prefix),
1067 )),
1068 ActiveBody::Step(id) => Some(format!(
1069 "context usage error in \"{method}\": the operation was called inside the \
1070 body of operation \"{id}\". A step body or a wait_for_condition check \
1071 cannot call durable operations; use run_in_child_context to group \
1072 operations."
1073 )),
1074 ActiveBody::Plugin => Some(format!(
1075 "context usage error in \"{method}\": the operation was called inside \
1076 plugin code. Plugin code cannot call durable operations."
1077 )),
1078 ActiveBody::Outcome(id) => Some(format!(
1079 "context usage error in \"{method}\": the operation was called while the \
1080 SDK records the outcome of operation \"{id}\". An error's Display and \
1081 source cannot call durable operations."
1082 )),
1083 ActiveBody::Unset => None,
1084 })
1085 .ok()
1086 .flatten();
1087 let message = message.or_else(|| {
1088 self.inner
1089 .task_ownership
1090 .check_current_task()
1091 .err()
1092 .map(|_| foreign_caller_message(method))
1093 });
1094 if let Some(message) = message {
1095 self.coordinator().record_fatal("Error".to_owned(), message);
1096 }
1097 self.mint_id()
1098 }
1099
1100 pub(crate) fn replay_span(&self) -> tracing::Span {
1106 self.inner.replay_span.clone()
1107 }
1108
1109 pub(crate) fn emit_operation_replayed(
1117 &self,
1118 wire_id: &str,
1119 operation_name: Option<&str>,
1120 operation_type: &str,
1121 operation_sub_type: Option<&str>,
1122 recorded_attempt: u32,
1123 ) {
1124 crate::tracing_layer::operation_replayed_event(&crate::tracing_layer::OperationIdentity {
1125 execution_arn: self.execution_arn(),
1126 request_id: &self.lambda_context().request_id,
1127 operation_id: wire_id,
1128 operation_name,
1129 operation_type,
1130 operation_sub_type,
1131 attempt: recorded_attempt.saturating_add(1),
1132 });
1133 }
1134
1135 pub(crate) fn new_flat_child(
1141 &self,
1142 child_positional_id: &str,
1143 parent_wire_id_override: &str,
1144 name: Option<&str>,
1145 ) -> Self {
1146 let engine = Arc::new(EngineState::new_child(
1147 child_positional_id,
1148 Arc::clone(&self.inner.engine.checkpoint_log),
1149 ));
1150 let replay_span = crate::tracing_layer::scoped_execution_span(
1155 &self.inner.execution_arn,
1156 &self.inner.lambda_context.request_id,
1157 self.inner.lambda_context.tenant_id.as_deref(),
1158 &crate::engine::compute_wire_id_public(child_positional_id),
1159 name,
1160 engine.is_replaying(),
1161 );
1162 Self {
1163 inner: Arc::new(Inner {
1164 execution_arn: self.inner.execution_arn.clone(),
1165 lambda_context: self.inner.lambda_context.clone(),
1166 engine,
1167 coordinator: Arc::clone(&self.inner.coordinator),
1168 task_ownership: Arc::clone(&self.inner.task_ownership),
1169 execution_client: self.inner.execution_client.clone(),
1170 checkpoint_token: Arc::clone(&self.inner.checkpoint_token),
1171 coalescer: Arc::clone(&self.inner.coalescer),
1172 #[cfg(test)]
1173 hooks: Arc::clone(&self.inner.hooks),
1174 parent_wire_id: Some(parent_wire_id_override.to_owned()),
1175 admission: Arc::clone(&self.inner.admission),
1176 owners: self.owners_with(child_positional_id),
1177 replay_span,
1178 plugins: self.inner.plugins.clone(),
1179 nesting_level: self.inner.nesting_level,
1180 preserve_child_depth: self.inner.preserve_child_depth,
1181 rebuilding: self.inner.rebuilding,
1182 }),
1183 awaited: None,
1184 }
1185 }
1186
1187 pub(crate) fn admission(&self) -> &Arc<Admission> {
1190 &self.inner.admission
1191 }
1192
1193 pub(crate) fn owners(&self) -> &[String] {
1195 &self.inner.owners
1196 }
1197
1198 pub(crate) async fn admit_operation(&self) {
1203 if !self.admission().admits(self.owners()) {
1204 std::future::pending::<()>().await;
1205 }
1206 }
1207
1208 fn owners_with(&self, positional_id: &str) -> Arc<[String]> {
1211 self.inner
1212 .owners
1213 .iter()
1214 .cloned()
1215 .chain(std::iter::once(crate::engine::compute_wire_id_public(
1216 positional_id,
1217 )))
1218 .collect()
1219 }
1220
1221 #[cfg(test)]
1224 pub(crate) fn set_pre_lock_gate(&self, gate: PreLockGate) {
1225 self.hooks().gate = Some(gate);
1226 }
1227
1228 #[cfg(test)]
1231 pub(crate) fn set_submission_recorder(&self, recorder: SubmissionRecorder) {
1232 self.hooks().submissions = Some(recorder);
1233 }
1234
1235 #[cfg(test)]
1238 pub(crate) fn set_settler_entry_recorder(&self, recorder: SettlerEntryRecorder) {
1239 self.hooks().settler_entries = Some(recorder);
1240 }
1241
1242 #[cfg(test)]
1244 fn hooks(&self) -> std::sync::MutexGuard<'_, TestHooks> {
1245 self.inner
1246 .hooks
1247 .lock()
1248 .unwrap_or_else(std::sync::PoisonError::into_inner)
1249 }
1250
1251 #[cfg(test)]
1254 async fn pass_gate(&self, updates: &[OperationUpdate]) {
1255 let gate = self.hooks().gate.clone();
1256 if let Some(notify) = gate.and_then(|gate| gate(updates)) {
1257 notify.notified().await;
1258 }
1259 }
1260
1261 #[cfg(test)]
1264 fn record_submission(&self, updates: &[OperationUpdate], write: &SubmittedWrite) {
1265 let recorder = self.hooks().submissions.clone();
1266 if let Some(recorder) = recorder
1267 && !updates.is_empty()
1268 {
1269 recorder(updates, &write.first_poll);
1270 }
1271 }
1272
1273 pub(crate) async fn await_retained_start(
1280 &self,
1281 wire_id: &str,
1282 start: Option<SubmittedWrite>,
1283 action: OperationAction,
1284 ) -> Result<(), OperationError> {
1285 #[cfg(test)]
1286 {
1287 let recorder = self.hooks().settler_entries.clone();
1288 if let Some(recorder) = recorder {
1289 recorder(wire_id, action.clone());
1290 }
1291 }
1292 #[cfg(not(test))]
1293 let _ = (wire_id, action);
1294 if let Some(start) = start
1295 && let Err(err) = start.await
1296 {
1297 return self.checkpoint_failure_unrecoverable(err).await;
1298 }
1299 Ok(())
1300 }
1301
1302 #[cfg(test)]
1305 pub(crate) fn buffer_unflushed(&self, updates: Vec<OperationUpdate>) {
1306 let tracked = self.track_updates(updates);
1307 self.inner
1308 .coalescer
1309 .join(self.admission(), self.coordinator(), self.owners(), tracked)
1310 .expect("the join is admitted");
1311 }
1312
1313 #[cfg(test)]
1316 pub(crate) fn queued_update_ids(&self) -> Vec<String> {
1317 self.inner.coalescer.pending_ids()
1318 }
1319
1320 #[cfg(test)]
1322 pub(crate) fn checkpoint_token_mutex(&self) -> Arc<Mutex<String>> {
1323 Arc::clone(&self.inner.checkpoint_token)
1324 }
1325
1326 pub(crate) fn plugins(&self) -> Option<&crate::plugin::InvocationPlugins> {
1329 self.inner.plugins.as_deref()
1330 }
1331
1332 pub(crate) async fn report_operation_changes(&self) {
1342 if let Some(plugins) = self.plugins() {
1343 plugins
1344 .changes_delivered(&self.inner.engine.checkpoint_log)
1345 .await;
1346 }
1347 }
1348
1349 pub(crate) fn shared_plugins(&self) -> Option<Arc<crate::plugin::InvocationPlugins>> {
1352 self.inner.plugins.clone()
1353 }
1354
1355 pub(crate) fn coordinator(&self) -> &Arc<InvocationCoordinator> {
1358 &self.inner.coordinator
1359 }
1360
1361 pub(crate) async fn hold_activity<F: Future>(&self, fut: F) -> F::Output {
1368 let Some(permit) = self.inner.coordinator.acquire(PermitKind::Activity) else {
1369 return std::future::pending().await;
1370 };
1371 let output = fut.await;
1372 drop(permit);
1373 output
1374 }
1375
1376 pub(crate) async fn wait_for_record(
1392 &self,
1393 wire_id: &str,
1394 ready: fn(&CheckpointStatus) -> bool,
1395 ) {
1396 let mut registration: Option<crate::coordinator::WaitRegistration> = None;
1397 std::future::poll_fn(|cx| {
1398 if let Some(live) = ®istration {
1399 live.update_waker(cx.waker());
1400 } else {
1401 let (status, due_at) = self
1405 .refresh_view(wire_id)
1406 .unwrap_or((CheckpointStatus::Started, None));
1407 match self.inner.coordinator.register_wait(
1408 wire_id,
1409 due_at,
1410 &status,
1411 self.awaited.clone(),
1412 cx.waker(),
1413 ) {
1414 Some(live) => registration = Some(live),
1415 None => return std::task::Poll::Pending,
1416 }
1417 }
1418 let is_ready = self
1419 .inner
1420 .engine
1421 .checkpoint_log
1422 .published_status_view(wire_id)
1423 .is_some_and(|view| ready(&view.status));
1424 if is_ready {
1425 registration = None;
1426 std::task::Poll::Ready(())
1427 } else {
1428 if let Some(live) = ®istration {
1429 live.resume_schedule();
1430 }
1431 std::task::Poll::Pending
1432 }
1433 })
1434 .await;
1435 self.report_operation_changes().await;
1439 }
1440
1441 pub(crate) fn refresh_view(
1444 &self,
1445 wire_id: &str,
1446 ) -> Option<(CheckpointStatus, Option<std::time::SystemTime>)> {
1447 self.inner
1448 .engine
1449 .checkpoint_log
1450 .with_record(wire_id, |record| (record.status.clone(), record.due_at))
1451 }
1452
1453 pub(crate) fn with_awaited(&self, flag: Arc<std::sync::atomic::AtomicBool>) -> Self {
1458 Self {
1459 inner: Arc::clone(&self.inner),
1460 awaited: Some(flag),
1461 }
1462 }
1463
1464 pub(crate) async fn wait_for_record_view(
1467 &self,
1468 positional_id: &str,
1469 ready: fn(&CheckpointStatus) -> bool,
1470 ) -> CheckpointStatusView {
1471 let wire_id = crate::engine::compute_wire_id_public(positional_id);
1472 self.wait_for_record(&wire_id, ready).await;
1473 #[expect(clippy::expect_used)]
1474 self.checkpoint_status_view(positional_id)
1478 .expect("a record that satisfied the wait is still in the log")
1479 }
1480
1481 pub(crate) fn remaining_time(&self) -> Option<Duration> {
1485 remaining_until(
1486 self.inner.lambda_context.deadline,
1487 std::time::SystemTime::now(),
1488 )
1489 }
1490
1491 pub(crate) fn task_ownership(&self) -> &Arc<TaskOwnership> {
1493 &self.inner.task_ownership
1494 }
1495
1496 pub(crate) fn enforce_task_ownership(&self) -> Result<(), OperationError> {
1499 self.inner
1500 .task_ownership
1501 .check_current_task()
1502 .map_err(|msg| {
1503 self.coordinator()
1504 .record_fatal("Error".to_owned(), foreign_caller_message("execute"));
1505 OperationError::from_kind(OperationErrorKind::Step(StepError::new(
1506 StepErrorKind::ExecutionFailed,
1507 Some(msg.into()),
1508 )))
1509 })
1510 }
1511
1512 pub(crate) fn with_checkpoint_record<R>(
1524 &self,
1525 positional_id: &str,
1526 f: impl FnOnce(&CheckpointRecord) -> R,
1527 ) -> Option<R> {
1528 let wire_id = crate::engine::compute_wire_id_public(positional_id);
1529 self.inner.engine.checkpoint_log.with_record(&wire_id, f)
1530 }
1531
1532 pub(crate) fn with_checkpoint_record_by_wire<R>(
1535 &self,
1536 wire_id: &str,
1537 f: impl FnOnce(&CheckpointRecord) -> R,
1538 ) -> Option<R> {
1539 self.inner.engine.checkpoint_log.with_record(wire_id, f)
1540 }
1541
1542 pub(crate) fn checkpoint_status_view(
1546 &self,
1547 positional_id: &str,
1548 ) -> Option<CheckpointStatusView> {
1549 let wire_id = crate::engine::compute_wire_id_public(positional_id);
1550 self.inner.engine.checkpoint_log.status_view(&wire_id)
1551 }
1552
1553 #[cfg(test)]
1560 pub(crate) fn has_checkpoint_record(&self, positional_id: &str) -> bool {
1561 let wire_id = crate::engine::compute_wire_id_public(positional_id);
1562 self.inner.engine.checkpoint_log.contains(&wire_id)
1563 }
1564
1565 pub(crate) fn checkpoint_result_payload(&self, positional_id: &str) -> Option<String> {
1572 self.with_checkpoint_record(positional_id, |record| record.result.clone())
1573 .flatten()
1574 }
1575
1576 pub(crate) fn checkpoint_callback_id(&self, positional_id: &str) -> Option<String> {
1579 self.with_checkpoint_record(positional_id, |record| record.callback_id.clone())
1580 .flatten()
1581 }
1582
1583 pub(crate) fn checkpoint_wire_error(
1587 &self,
1588 positional_id: &str,
1589 ) -> Option<crate::error::WireError> {
1590 self.with_checkpoint_record(positional_id, |record| {
1591 crate::error::WireError::new(record.error_type.clone(), record.error_message.clone())
1592 .with_error_data(record.error_data.clone())
1593 .with_stack_trace(record.stack_trace.clone().unwrap_or_default())
1594 })
1595 }
1596
1597 pub(crate) fn checkpoint_terminal_replay(
1606 &self,
1607 positional_id: &str,
1608 ) -> Option<TerminalReplaySnapshot> {
1609 self.with_checkpoint_record(positional_id, |record| TerminalReplaySnapshot {
1610 status: record.status.clone(),
1611 replay_children: record.replay_children,
1612 result: record.result.clone(),
1613 error_message: record.error_message.clone(),
1614 error_type: record.error_type.clone(),
1615 error_data: record.error_data.clone(),
1616 stack_trace: record.stack_trace.clone(),
1617 })
1618 }
1619
1620 pub(crate) fn checkpoint_view_validated(
1633 &self,
1634 positional_id: &str,
1635 wire_id: &str,
1636 claimed_type: &str,
1637 claimed_sub_type: Option<&str>,
1638 claimed_name: Option<&str>,
1639 ) -> Result<Option<CheckpointStatusView>, OperationError> {
1640 let Some((mismatch, view)) = self.with_checkpoint_record(positional_id, |record| {
1641 (
1642 replay_identity_mismatch(record, claimed_type, claimed_sub_type, claimed_name),
1643 CheckpointStatusView {
1644 status: record.status.clone(),
1645 attempt: record.attempt,
1646 replay_children: record.replay_children,
1647 },
1648 )
1649 }) else {
1650 return Ok(None);
1651 };
1652 if let Some(expected) = mismatch {
1653 return Err(self.replay_mismatch_error(
1655 wire_id,
1656 &expected,
1657 claimed_type,
1658 claimed_sub_type,
1659 claimed_name,
1660 ));
1661 }
1662 if let CheckpointStatus::Unknown(raw) = &view.status {
1663 return Err(self.unrecognized_status_error(wire_id, raw));
1668 }
1669 Ok(Some(view))
1670 }
1671
1672 pub(crate) fn preflight_replay_identity(
1690 &self,
1691 op_id: &OperationId,
1692 claimed_type: &str,
1693 claimed_sub_type: Option<&str>,
1694 claimed_name: Option<&str>,
1695 ) -> Result<(), OperationError> {
1696 if !self.admission().admits(self.owners()) {
1703 return Ok(());
1704 }
1705 self.checkpoint_view_validated(
1706 op_id.positional(),
1707 op_id.wire(),
1708 claimed_type,
1709 claimed_sub_type,
1710 claimed_name,
1711 )?;
1712 Ok(())
1713 }
1714
1715 #[cfg(test)]
1733 pub(crate) fn validate_replay_identity(
1734 &self,
1735 record: &CheckpointRecord,
1736 wire_id: &str,
1737 claimed_type: &str,
1738 claimed_sub_type: Option<&str>,
1739 claimed_name: Option<&str>,
1740 ) -> Result<(), OperationError> {
1741 match replay_identity_mismatch(record, claimed_type, claimed_sub_type, claimed_name) {
1742 None => Ok(()),
1743 Some(expected) => Err(self.replay_mismatch_error(
1744 wire_id,
1745 &expected,
1746 claimed_type,
1747 claimed_sub_type,
1748 claimed_name,
1749 )),
1750 }
1751 }
1752
1753 fn replay_mismatch_error(
1764 &self,
1765 wire_id: &str,
1766 expected: &str,
1767 claimed_type: &str,
1768 claimed_sub_type: Option<&str>,
1769 claimed_name: Option<&str>,
1770 ) -> OperationError {
1771 let err = OperationError::from_kind(OperationErrorKind::NonDeterministicExecution(
1772 NonDeterministicExecutionError::from_kind(
1773 NonDeterministicExecutionErrorKind::OperationMismatch(
1774 crate::error::OperationMismatch::new(
1775 wire_id,
1776 expected,
1777 format_op_identity(claimed_type, claimed_sub_type, claimed_name),
1778 ),
1779 ),
1780 ),
1781 ));
1782 self.inner.coordinator.record_fatal(
1783 "NonDeterministicExecutionError".to_owned(),
1784 crate::error::chain_string(&err),
1785 );
1786 err
1787 }
1788
1789 pub(crate) fn unrecognized_status_error(&self, wire_id: &str, raw: &str) -> OperationError {
1799 let err = OperationError::from_kind(OperationErrorKind::NonDeterministicExecution(
1800 NonDeterministicExecutionError::from_kind(
1801 NonDeterministicExecutionErrorKind::UnrecognizedStatus(
1802 crate::error::UnrecognizedStatus::new(wire_id, raw),
1803 ),
1804 ),
1805 ));
1806 self.inner.coordinator.record_fatal(
1807 "NonDeterministicExecutionError".to_owned(),
1808 crate::error::chain_string(&err),
1809 );
1810 err
1811 }
1812
1813 pub(crate) async fn checkpoint_failure_unrecoverable<T: NoRuntimeOutcome>(
1827 &self,
1828 err: ExecutionClientError,
1829 ) -> T {
1830 if let Some(no_runtime) = std::error::Error::source(&err)
1831 .and_then(|source| source.downcast_ref::<crate::task::NoRuntime>())
1832 {
1833 return T::no_runtime(*no_runtime);
1834 }
1835 std::future::pending::<T>().await
1836 }
1837
1838 fn record_write_failure(&self, op_wire_id: &str, err: &ExecutionClientError) {
1848 if err.is_retryable() {
1849 tracing::error!(
1850 operation_id = %op_wire_id,
1851 error = %err,
1852 "retryable checkpoint failure exhausted retries; failing the invocation \
1853 so the service re-invokes"
1854 );
1855 } else {
1856 tracing::error!(
1857 operation_id = %op_wire_id,
1858 error = %err,
1859 "non-retryable checkpoint failure; failing the execution"
1860 );
1861 }
1862 self.inner
1863 .coordinator
1864 .fail_write(checkpoint_failure_fatal(err), || {
1865 self.abandon_queued_writes();
1866 });
1867 }
1868
1869 pub(crate) async fn serdes_fault<T>(
1881 &self,
1882 call: &'static str,
1883 operation_id: &str,
1884 name: Option<&str>,
1885 err: BoxError,
1886 ) -> T {
1887 let detail = crate::error::wire_message(&*err);
1888 let message = match name {
1889 Some(name) => {
1890 format!("serdes {call} failed for operation \"{name}\" ({operation_id}): {detail}")
1891 }
1892 None => format!("serdes {call} failed for operation {operation_id}: {detail}"),
1893 };
1894 tracing::error!(
1895 operation_id = %operation_id,
1896 error = %detail,
1897 "serdes {call} failed with a retryable error; failing the invocation so the \
1898 service re-invokes"
1899 );
1900 self.inner
1901 .coordinator
1902 .record_invocation_fault(crate::error::SERDES_FAILED_ERROR_TYPE, message);
1903 std::future::pending::<T>().await
1904 }
1905
1906 pub(crate) async fn route_serdes_error(
1914 &self,
1915 call: &'static str,
1916 operation_id: &str,
1917 name: Option<&str>,
1918 err: BoxError,
1919 ) -> BoxError {
1920 if crate::serdes::is_retryable_serdes_error(&*err) {
1921 return self.serdes_fault(call, operation_id, name, err).await;
1922 }
1923 err
1924 }
1925
1926 pub(crate) async fn checkpoint_updates(
1934 &self,
1935 updates: Vec<OperationUpdate>,
1936 ) -> Result<CheckpointOutput, ExecutionClientError> {
1937 self.submit_checkpoint_updates(updates).await
1938 }
1939
1940 pub(crate) fn submit_checkpoint_updates(
1947 &self,
1948 updates: Vec<OperationUpdate>,
1949 ) -> SubmittedWrite {
1950 self.checkpoint_with_urgency(updates, false)
1951 }
1952
1953 pub(crate) fn submit_checkpoint_updates_urgent(
1967 &self,
1968 updates: Vec<OperationUpdate>,
1969 ) -> SubmittedWrite {
1970 self.checkpoint_with_urgency(updates, true)
1971 }
1972
1973 pub(crate) fn reserve_checkpoint_updates(
1981 &self,
1982 updates: Vec<OperationUpdate>,
1983 ) -> ReservedWrite {
1984 #[cfg(test)]
1985 let recorded = updates.clone();
1986 let tracked = self.track_updates(updates);
1987 let coalescer = Arc::clone(&self.inner.coalescer);
1988 let Some(batch) = self.join_batch(&coalescer, tracked, false, None, true) else {
1989 return ReservedWrite::rejected();
1990 };
1991 let write = batch_result(batch).reported_by(self);
1992 #[cfg(test)]
1993 self.record_submission(&recorded, &write);
1994 ReservedWrite {
1995 write,
1996 hold: Some(coalescer),
1997 }
1998 }
1999
2000 pub(crate) async fn checkpoint_refresh(&self, first_wire_id: &str) {
2009 let write = self.write_with_urgency(Vec::new(), true, Some(first_wire_id));
2010 drop(write.reported_by(self).await);
2015 }
2016
2017 fn checkpoint_with_urgency(
2022 &self,
2023 updates: Vec<OperationUpdate>,
2024 urgent: bool,
2025 ) -> SubmittedWrite {
2026 let tracked = self.track_updates(updates);
2042 self.write_with_urgency(tracked, urgent, None)
2043 .reported_by(self)
2044 }
2045
2046 fn track_updates(&self, updates: Vec<OperationUpdate>) -> Vec<TrackedUpdate> {
2049 updates
2050 .into_iter()
2051 .map(|update| {
2052 let attempt = self
2055 .inner
2056 .engine
2057 .checkpoint_log
2058 .status_view(update.id())
2059 .map_or(0, |view| view.attempt)
2060 .saturating_add(1);
2061 TrackedUpdate::capture(update, attempt)
2062 })
2063 .collect()
2064 }
2065
2066 fn write_with_urgency(
2085 &self,
2086 updates: Vec<TrackedUpdate>,
2087 urgent: bool,
2088 refresh_id: Option<&str>,
2089 ) -> SubmittedWrite {
2090 #[cfg(test)]
2094 let flag = {
2095 let flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
2096 let recorder = self.hooks().submissions.clone();
2097 if let Some(recorder) = recorder
2098 && !updates.is_empty()
2099 && self.admission().admits(self.owners())
2100 {
2101 let submitted: Vec<OperationUpdate> =
2102 updates.iter().map(|t| t.update.clone()).collect();
2103 recorder(&submitted, &flag);
2104 }
2105 flag
2106 };
2107 #[cfg_attr(not(test), expect(unused_mut))] let mut write = self.write_with_urgency_unrecorded(updates, urgent, refresh_id);
2109 #[cfg(test)]
2110 {
2111 write.first_poll = flag;
2112 }
2113 write
2114 }
2115
2116 fn write_with_urgency_unrecorded(
2118 &self,
2119 updates: Vec<TrackedUpdate>,
2120 urgent: bool,
2121 refresh_id: Option<&str>,
2122 ) -> SubmittedWrite {
2123 let coalescer = Arc::clone(&self.inner.coalescer);
2124 match self.join_batch(&coalescer, updates, urgent, refresh_id, false) {
2125 Some(batch) => batch_result(batch),
2126 None => SubmittedWrite::rejected(),
2127 }
2128 }
2129
2130 fn join_batch(
2135 &self,
2136 coalescer: &Arc<CheckpointCoalescer>,
2137 updates: Vec<TrackedUpdate>,
2138 urgent: bool,
2139 refresh_id: Option<&str>,
2140 reserve: bool,
2141 ) -> Option<Arc<CheckpointBatch>> {
2142 let (batch, opened) = coalescer.join_owned(
2153 self.admission(),
2154 self.coordinator(),
2155 self.owners(),
2156 updates,
2157 refresh_id,
2158 reserve,
2159 )?;
2160 if urgent || coalescer.delay().is_zero() {
2161 self.spawn_batch_flush(coalescer, &batch, Duration::ZERO);
2162 } else if opened {
2163 self.spawn_batch_flush(coalescer, &batch, coalescer.delay());
2167 }
2168 Some(batch)
2169 }
2170
2171 async fn write_tracked_direct(
2188 &self,
2189 tracked: Vec<TrackedUpdate>,
2190 ) -> Result<CheckpointOutput, WriteError> {
2191 let mut updates = Vec::with_capacity(tracked.len());
2192 let mut events = Vec::with_capacity(tracked.len());
2193 for TrackedUpdate { update, event } in tracked {
2194 updates.push(update);
2195 events.extend(event);
2196 }
2197 self.write_unchecked(updates, &events, None).await
2198 }
2199
2200 async fn write_unchecked(
2204 &self,
2205 updates: Vec<OperationUpdate>,
2206 events: &[crate::tracing_layer::PendingTransitionEvent],
2207 refresh_id: Option<&str>,
2208 ) -> Result<CheckpointOutput, WriteError> {
2209 self.checkpoint_direct_with_events(updates, events, refresh_id)
2210 .await
2211 }
2212
2213 fn spawn_batch_flush(
2228 &self,
2229 coalescer: &Arc<CheckpointCoalescer>,
2230 batch: &Arc<CheckpointBatch>,
2231 delay: Duration,
2232 ) {
2233 let ctx = self.clone();
2234 let claim_from = Arc::clone(coalescer);
2235 let coalescer = Arc::clone(coalescer);
2236 let publish_to = Arc::clone(batch);
2237 let batch = Arc::clone(batch);
2238 let flush = async move {
2249 if !delay.is_zero() {
2250 ctx.coordinator().coalescing_window(delay).await;
2256 }
2257 let _writer = coalescer.writer_lock().lock().await;
2258 if let Some((updates, permits)) = coalescer.take_batch_unheld(&batch).await {
2259 if let Ok(output) = ctx
2263 .write_batched_updates(updates, coalescer.limits(), batch.refresh_id())
2264 .await
2265 {
2266 batch.publish(Ok(output));
2267 }
2268 drop(permits);
2269 }
2270 };
2271 let spawned = crate::task::spawn_detached(self.coordinator(), flush);
2272 let Err(no_runtime) = spawned else {
2283 return;
2284 };
2285 #[cfg(test)]
2286 crate::coordinator::race_points::reach(
2287 crate::coordinator::race_points::RacePoint::BatchNoRuntimeClaim,
2288 );
2289 if let Some((updates, permits)) =
2290 claim_from.claim_unspawned(&publish_to, self.coordinator())
2291 {
2292 drop(updates);
2293 publish_to.publish(Err(ExecutionClientError::with_source(
2294 ExecutionClientErrorScope::Invocation,
2295 no_runtime.to_string(),
2296 no_runtime,
2297 )));
2298 drop(permits);
2299 }
2300 }
2301
2302 async fn write_batched_updates(
2319 &self,
2320 updates: Vec<TrackedUpdate>,
2321 limits: BatchLimits,
2322 refresh_id: Option<String>,
2323 ) -> Result<CheckpointOutput, WriteError> {
2324 if updates.is_empty() {
2325 return self
2327 .write_unchecked(Vec::new(), &[], refresh_id.as_deref())
2328 .await;
2329 }
2330 let mut last = None;
2331 for chunk in split_into_requests(updates, &limits) {
2332 last = Some(self.write_tracked_direct(chunk).await?);
2333 }
2334 #[expect(clippy::expect_used)]
2335 Ok(last.expect("a non-empty batch has a request"))
2337 }
2338
2339 pub(crate) async fn flush_pending_checkpoints(&self) {
2361 let coalescer = &self.inner.coalescer;
2362 let _writer = coalescer.writer_lock().lock().await;
2366 while let Some((batch, updates, permits)) = coalescer.take_any_unheld().await {
2367 let written = self
2368 .write_batched_updates(updates, coalescer.limits(), batch.refresh_id())
2369 .await;
2370 drop(permits);
2371 match written {
2372 Ok(output) => batch.publish(Ok(output)),
2373 Err(_) => break,
2374 }
2375 }
2376 }
2377
2378 #[cfg(test)]
2382 pub(crate) async fn checkpoint_updates_direct(
2383 &self,
2384 updates: Vec<OperationUpdate>,
2385 ) -> Result<CheckpointOutput, ExecutionClientError> {
2386 let result = self.submit_checkpoint_updates_urgent(updates).await;
2387 self.report_operation_changes().await;
2388 result
2389 }
2390
2391 pub(crate) async fn checkpoint_updates_after_drive(
2402 &self,
2403 updates: Vec<OperationUpdate>,
2404 ) -> Result<CheckpointOutput, WriteError> {
2405 let coalescer = &self.inner.coalescer;
2406 let _writer = coalescer.writer_lock().lock().await;
2407 let tracked = updates.into_iter().map(TrackedUpdate::untracked).collect();
2408 self.write_batched_updates(tracked, coalescer.limits(), None)
2409 .await
2410 }
2411
2412 async fn checkpoint_direct_with_events(
2436 &self,
2437 updates: Vec<OperationUpdate>,
2438 events: &[crate::tracing_layer::PendingTransitionEvent],
2439 refresh_id: Option<&str>,
2440 ) -> Result<CheckpointOutput, WriteError> {
2441 #[cfg(test)]
2442 self.pass_gate(&updates).await;
2443 let failure_id = updates
2444 .first()
2445 .map(|update| update.id().to_owned())
2446 .or_else(|| refresh_id.map(str::to_owned))
2447 .unwrap_or_default();
2448 let token_guard = self.inner.checkpoint_token.lock().await;
2453 if self.inner.coordinator.withdrawn() || self.inner.coordinator.write_failed() {
2456 return Err(WriteError::Stopped);
2457 }
2458 let Some(client) = self.inner.execution_client.as_ref() else {
2459 let err = ExecutionClientError::new_non_retryable("no execution client configured");
2462 self.record_write_failure(&failure_id, &err);
2463 drop(token_guard);
2464 return Err(WriteError::Failed(err));
2465 };
2466 self.write_locked(client, token_guard, updates, events, &failure_id)
2467 .await
2468 }
2469
2470 async fn write_locked(
2474 &self,
2475 client: &Arc<dyn ExecutionClient>,
2476 mut token_guard: tokio::sync::MutexGuard<'_, String>,
2477 updates: Vec<OperationUpdate>,
2478 events: &[crate::tracing_layer::PendingTransitionEvent],
2479 failure_id: &str,
2480 ) -> Result<CheckpointOutput, WriteError> {
2481 let retried = wfc_retries(&updates);
2482 let has_execution_update = updates
2485 .iter()
2486 .any(|update| update.r#type() == &aws_sdk_lambda::types::OperationType::Execution);
2487 let output = match client
2488 .checkpoint(&self.inner.execution_arn, &token_guard, updates)
2489 .await
2490 {
2491 Ok(output) => output,
2492 Err(err) => {
2493 self.record_write_failure(failure_id, &err);
2494 return Err(WriteError::Failed(err));
2495 }
2496 };
2497 let next_token = output.checkpoint_token().map(ToOwned::to_owned);
2498 let tokenless = next_token.is_none();
2499
2500 for event in events {
2504 event.emit(self.execution_arn(), &self.lambda_context().request_id);
2505 }
2506
2507 match next_token {
2508 Some(next_token) => *token_guard = next_token,
2510 None if has_execution_update => {}
2511 None => {
2512 self.inner.coordinator.withdraw(
2519 self.execution_arn(),
2520 &self.lambda_context().request_id,
2521 || self.abandon_queued_writes(),
2522 );
2523 return Err(WriteError::Stopped);
2524 }
2525 }
2526
2527 let pages = if output.next_marker.is_some() && !tokenless {
2537 Some(
2538 crate::client::get_full_state(
2539 client.as_ref(),
2540 &self.inner.execution_arn,
2541 &token_guard,
2542 )
2543 .await,
2544 )
2545 } else {
2546 None
2547 };
2548
2549 let publication = self.inner.engine.checkpoint_log.publish();
2558 let mut merged: Vec<String> = Vec::new();
2559 let mut change_set = ChangeSet::default();
2564 if !output.updated_operations.is_empty() {
2565 let changed_ids = crate::client::merge_operations_into_log(
2566 &self.inner.engine.checkpoint_log,
2567 &output.updated_operations,
2568 );
2569 self.collect_changes(&mut change_set, changed_ids, &output.updated_operations);
2570 merged.extend(
2571 output
2572 .updated_operations
2573 .iter()
2574 .map(|op| op.id().to_owned()),
2575 );
2576 }
2577 match pages {
2578 Some(Ok(full_state)) => {
2579 if !full_state.is_empty() {
2580 let changed_ids = crate::client::merge_operations_into_log(
2581 &self.inner.engine.checkpoint_log,
2582 &full_state,
2583 );
2584 self.collect_changes(&mut change_set, changed_ids, &full_state);
2585 merged.extend(full_state.iter().map(|op| op.id().to_owned()));
2586 }
2587 }
2588 Some(Err(err)) => {
2589 self.queue_changes(change_set);
2592 drop(publication);
2593 self.record_write_failure(failure_id, &err);
2594 return Err(WriteError::Failed(err));
2595 }
2596 None => {}
2597 }
2598 self.queue_changes(change_set);
2601 drop(publication);
2602
2603 for (wire_id, payload) in &retried {
2608 if !merged.iter().any(|id| id == wire_id) {
2609 mark_retry_pending(
2610 &self.inner.engine.checkpoint_log,
2611 wire_id,
2612 payload.as_deref(),
2613 );
2614 }
2615 }
2616
2617 drop(token_guard);
2622 self.inner.coordinator.wake_operations(&merged);
2623
2624 Ok(output)
2625 }
2626
2627 fn abandon_queued_writes(&self) {
2633 drop(self.inner.coalescer.abandon_pending());
2634 }
2635
2636 fn collect_changes(
2642 &self,
2643 change_set: &mut ChangeSet,
2644 changed_ids: Vec<String>,
2645 operations: &[aws_sdk_lambda::types::Operation],
2646 ) {
2647 let Some(plugins) = self.plugins() else {
2648 return;
2649 };
2650 for id in changed_ids {
2651 if !change_set.ids.contains(&id) {
2652 change_set.ids.push(id);
2653 }
2654 }
2655 for info in operations
2656 .iter()
2657 .filter_map(crate::client::execution_info_from_operation)
2658 {
2659 if plugins.replace_execution(info.clone()) {
2660 change_set.executions.retain(|held| held.id() != info.id());
2661 change_set.executions.push(info);
2662 }
2663 }
2664 }
2665
2666 fn queue_changes(&self, change_set: ChangeSet) {
2670 let Some(plugins) = self.plugins() else {
2671 return;
2672 };
2673 let log = &self.inner.engine.checkpoint_log;
2674 let mut updated: Vec<crate::plugin::OperationInfo> = change_set
2675 .ids
2676 .iter()
2677 .filter_map(|id| {
2678 log.with_record(id, |record| {
2679 crate::plugin::OperationInfo::from_record(id, record)
2680 })
2681 })
2682 .collect();
2683 updated.extend(change_set.executions);
2684 if !updated.is_empty() {
2685 plugins.push_operation_change(updated);
2686 }
2687 }
2688
2689 pub(crate) fn parent_wire_id(&self) -> Option<&str> {
2691 self.inner.parent_wire_id.as_deref()
2692 }
2693
2694 pub(crate) fn get_attempt(&self, op_id: &OperationId) -> u32 {
2697 self.checkpoint_status_view(op_id.positional())
2700 .map_or(0, |view| view.attempt)
2701 }
2702
2703 pub fn execution_arn(&self) -> &str {
2722 &self.inner.execution_arn
2723 }
2724
2725 pub fn lambda_context(&self) -> &lambda_runtime::Context {
2745 &self.inner.lambda_context
2746 }
2747
2748 pub fn is_replaying(&self) -> bool {
2774 self.inner.engine.is_replaying()
2775 }
2776
2777 pub fn step<O, F, Fut>(&self, f: F) -> StepBuilder<O, F, Fut>
2807 where
2808 F: FnOnce(StepContext) -> Fut + Clone + Send + 'static,
2809 Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
2810 O: Send + 'static,
2811 {
2812 let op_id = self.claim_operation_id("step");
2813 StepBuilder::new_internal(self.clone(), op_id, f)
2816 }
2817
2818 pub fn wait(&self, duration: Duration) -> WaitBuilder {
2847 #[expect(clippy::cast_possible_truncation)]
2850 let secs = (duration.as_secs_f64().ceil() as i64).clamp(1, i64::from(i32::MAX)) as i32;
2852 let op_id = self.claim_operation_id("wait");
2853 WaitBuilder::new_internal(self.clone(), op_id, secs)
2854 }
2855
2856 pub fn invoke<O>(
2911 &self,
2912 function_id: impl Into<String>,
2913 input: impl Serialize + Send + 'static,
2914 ) -> InvokeBuilder<O, impl Serialize + Send + 'static>
2915 where
2916 O: Send + 'static,
2917 {
2918 let op_id = self.claim_operation_id("invoke");
2919 InvokeBuilder::new_internal(self.clone(), op_id, function_id.into(), input)
2923 }
2924
2925 pub fn run_in_child_context<O, F, Fut>(&self, f: F) -> ChildBuilder<O, F, Fut>
2962 where
2963 F: FnOnce(DurableContext) -> Fut + Send + 'static,
2964 Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
2965 O: Send + 'static,
2966 {
2967 let op_id = self.claim_operation_id("run_in_child_context");
2968 ChildBuilder::new_internal(self.clone(), op_id, f)
2973 }
2974
2975 pub fn with_retry<O, F, Fut>(&self, f: F) -> WithRetryBuilder<O, F, Fut>
3045 where
3046 F: Fn(DurableContext, u32) -> Fut + Send + Sync + 'static,
3047 Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
3048 O: Send + 'static,
3049 {
3050 WithRetryBuilder::new_internal(self.clone(), f)
3053 }
3054
3055 pub fn wait_for_condition<S, F, Fut>(
3124 &self,
3125 check: F,
3126 initial_state: S,
3127 ) -> WaitForConditionBuilder<S, F, Fut>
3128 where
3129 F: Fn(StepContext, S) -> Fut + Send + Sync + 'static,
3130 Fut: Future<Output = Result<S, BoxError>> + Send + 'static,
3131 S: Clone + Send + Sync + 'static,
3132 {
3133 let op_id = self.claim_operation_id("wait_for_condition");
3134 WaitForConditionBuilder::new_internal(self.clone(), op_id, initial_state, check)
3135 }
3136
3137 pub fn create_callback<O>(&self) -> CreateCallbackBuilder<O>
3165 where
3166 O: Send + 'static,
3167 {
3168 let op_id = self.claim_operation_id("create_callback");
3169 CreateCallbackBuilder::new_internal(self.clone(), op_id)
3170 }
3171
3172 pub fn wait_for_callback<O>(
3239 &self,
3240 submitter: impl Submitter,
3241 ) -> WaitForCallbackBuilder<O, impl Submitter>
3242 where
3243 O: Send + 'static,
3244 {
3245 let op_id = self.claim_operation_id("wait_for_callback");
3246 WaitForCallbackBuilder::new_internal(self.clone(), op_id, submitter)
3247 }
3248
3249 pub fn map<Items, I, O, F, Fut>(&self, items: Items, f: F) -> MapBuilder<I, O, F, Fut>
3345 where
3346 Items: IntoIterator<Item = I>,
3347 F: Fn(DurableContext, I, usize) -> Fut + Send + Sync + 'static,
3348 Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
3349 I: Send + 'static,
3350 O: Send + 'static,
3351 {
3352 let op_id = self.claim_operation_id("map");
3353 let items: Vec<I> = items.into_iter().collect();
3359 MapBuilder::new_internal(self.clone(), op_id, items, f)
3360 }
3361
3362 pub fn parallel<Branches, O>(&self, branches: Branches) -> ParallelBuilder<O>
3413 where
3414 Branches: IntoIterator<Item = Branch<O>>,
3415 O: Send + 'static,
3416 {
3417 let op_id = self.claim_operation_id("parallel");
3418 let branch_tuples: Vec<_> = branches.into_iter().map(Branch::into_parts).collect();
3419 ParallelBuilder::new_internal(self.clone(), op_id, branch_tuples)
3420 }
3421
3422 pub fn parallel_mixed(&self, branches: crate::BranchSet) -> ParallelMixedBuilder {
3517 let op_id = self.claim_operation_id("parallel_mixed");
3518 ParallelMixedBuilder::new_internal(self.clone(), op_id, branches)
3519 }
3520
3521 pub fn try_join_all<O>(
3607 &self,
3608 futures: impl IntoIterator<Item = DurableFuture<O>>,
3609 ) -> TryJoinAllBuilder<O>
3610 where
3611 O: Serialize + DeserializeOwned + Send + 'static,
3612 {
3613 TryJoinAllBuilder::new_internal(self.clone(), futures.into_iter().collect())
3614 }
3615
3616 pub fn join_all<O>(
3718 &self,
3719 futures: impl IntoIterator<Item = DurableFuture<O>>,
3720 ) -> JoinAllBuilder<O>
3721 where
3722 O: Serialize + DeserializeOwned + Send + 'static,
3723 {
3724 JoinAllBuilder::new_internal(self.clone(), futures.into_iter().collect())
3725 }
3726
3727 pub fn select_ok<O>(
3818 &self,
3819 futures: impl IntoIterator<Item = DurableFuture<O>>,
3820 ) -> SelectOkBuilder<O>
3821 where
3822 O: Serialize + DeserializeOwned + Send + 'static,
3823 {
3824 SelectOkBuilder::new_internal(self.clone(), futures.into_iter().collect())
3825 }
3826
3827 pub fn race<O>(&self, futures: impl IntoIterator<Item = DurableFuture<O>>) -> RaceBuilder<O>
3920 where
3921 O: Serialize + DeserializeOwned + Send + 'static,
3922 {
3923 RaceBuilder::new_internal(self.clone(), futures.into_iter().collect())
3924 }
3925}
3926
3927#[derive(Debug, Clone)]
3955#[non_exhaustive]
3956pub struct StepContext {
3957 attempt: u32,
3958 _private: (),
3959}
3960
3961impl StepContext {
3962 pub(crate) fn new(attempt: u32) -> Self {
3964 Self {
3965 attempt,
3966 _private: (),
3967 }
3968 }
3969
3970 pub fn attempt(&self) -> u32 {
3973 self.attempt
3974 }
3975}
3976
3977pub(crate) fn canonical_op_type(raw: &str) -> String {
3988 let mut screaming = String::with_capacity(raw.len() + 4);
3992 let mut prev_is_lower = false;
3993 for ch in raw.chars() {
3994 if ch.is_ascii_uppercase() && prev_is_lower {
3995 screaming.push('_');
3996 }
3997 prev_is_lower = ch.is_ascii_lowercase();
3998 screaming.push(ch.to_ascii_uppercase());
3999 }
4000 aws_sdk_lambda::types::OperationType::from(screaming.as_str())
4003 .as_str()
4004 .to_owned()
4005}
4006
4007fn replay_identity_mismatch(
4017 record: &CheckpointRecord,
4018 claimed_type: &str,
4019 claimed_sub_type: Option<&str>,
4020 claimed_name: Option<&str>,
4021) -> Option<String> {
4022 let expected_type = record.op_type.as_deref()?;
4027
4028 let mismatch = || {
4029 Some(format_op_identity(
4030 expected_type,
4031 record.sub_type.as_deref(),
4032 record.op_name.as_deref(),
4033 ))
4034 };
4035
4036 if canonical_op_type(expected_type) != canonical_op_type(claimed_type) {
4041 return mismatch();
4042 }
4043
4044 let sub_type_matches = match (record.sub_type.as_deref(), claimed_sub_type) {
4049 (Some(expected), Some(claimed)) => expected.eq_ignore_ascii_case(claimed),
4050 (None, None) => true,
4051 (Some(_), None) | (None, Some(_)) => false,
4052 };
4053 if !sub_type_matches {
4054 return mismatch();
4055 }
4056
4057 let claimed_name = claimed_name.filter(|n| !n.is_empty());
4069 let expected_name = record.op_name.as_deref().filter(|n| !n.is_empty());
4070 let name_matches = match (expected_name, claimed_name) {
4071 (Some(expected), Some(claimed)) => expected == claimed,
4072 (None, None) => true,
4073 (Some(_), None) | (None, Some(_)) => false,
4074 };
4075 if !name_matches {
4076 return mismatch();
4077 }
4078
4079 None
4080}
4081
4082fn format_op_identity(op_type: &str, sub_type: Option<&str>, name: Option<&str>) -> String {
4085 let mut s = String::with_capacity(64);
4086 s.push_str(op_type);
4087 if let Some(sub) = sub_type {
4088 s.push('/');
4089 s.push_str(sub);
4090 }
4091 if let Some(n) = name {
4092 s.push_str(" named \"");
4093 s.push_str(n);
4094 s.push('"');
4095 }
4096 s
4097}
4098
4099pub(crate) trait NoRuntimeOutcome {
4102 fn no_runtime(no_runtime: crate::task::NoRuntime) -> Self;
4104}
4105
4106impl<T> NoRuntimeOutcome for Result<T, OperationError> {
4107 fn no_runtime(no_runtime: crate::task::NoRuntime) -> Self {
4108 Err(crate::task::NoRuntime::operation_error(no_runtime))
4109 }
4110}
4111
4112impl NoRuntimeOutcome for OperationError {
4114 fn no_runtime(no_runtime: crate::task::NoRuntime) -> Self {
4115 crate::task::NoRuntime::operation_error(no_runtime)
4116 }
4117}
4118
4119#[cfg(test)]
4120#[expect(clippy::disallowed_methods)]
4123mod tests {
4124 use super::*;
4125 use crate::client::{
4126 GetStateOutput, InMemoryExecutionClient, TestResponse, operations_to_checkpoint_log,
4127 };
4128 use crate::engine::{CheckpointLog, CheckpointStatus};
4129 use aws_sdk_lambda::types::Operation;
4130 use std::sync::Arc;
4131
4132 #[tokio::test]
4136 async fn typed_retryable_serdes_error_fault_omits_the_type_name() {
4137 let typed = crate::TypedError::with_type(
4138 "Throttled",
4139 crate::serdes::RetryableSerdesError::new("mount timed out"),
4140 );
4141 assert!(crate::serdes::is_retryable_serdes_error(&typed));
4142 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
4143 let ctx = DurableContext::new_root_with_client(
4144 "arn:test".to_owned(),
4145 lambda_runtime::Context::default(),
4146 Arc::new(CheckpointLog::empty()),
4147 Arc::clone(&client) as Arc<dyn ExecutionClient>,
4148 "token0".to_owned(),
4149 );
4150 let fault_ctx = ctx.clone();
4151 let outcome = crate::driver::test_support::outcome_of(
4152 ctx.clone(),
4153 Arc::clone(ctx.task_ownership()),
4154 Arc::clone(ctx.coordinator()),
4155 async move {
4156 let _ = fault_ctx
4157 .route_serdes_error("serialize", "op-1", Some("s"), Box::new(typed))
4158 .await;
4159 },
4160 )
4161 .await;
4162 let crate::driver::InvocationOutcome::Fault {
4163 error_type,
4164 message,
4165 } = outcome
4166 else {
4167 unreachable!("expected an invocation fault, got {outcome:?}");
4168 };
4169 assert_eq!(error_type, crate::error::SERDES_FAILED_ERROR_TYPE);
4170 assert_eq!(
4171 message,
4172 "serdes serialize failed for operation \"s\" (op-1): \
4173 retryable serdes failure: mount timed out"
4174 );
4175 }
4176
4177 #[test]
4181 fn nesting_levels_follow_constructors() {
4182 let (root, _client) = DurableContext::test_context_with_depth(5);
4183 let level = |ctx: &DurableContext| ctx.inner.nesting_level;
4184 assert_eq!(level(&root), 0);
4185 assert_eq!(root.inner.preserve_child_depth, 5);
4186
4187 let batch = root.new_child("1", None);
4188 let normal = batch.new_child("1-1", None);
4189 let flat = batch.new_flat_child("1-2", "parent", None);
4190 assert_eq!((level(&batch), level(&normal), level(&flat)), (1, 2, 1));
4191
4192 let child = root.new_child("2", None);
4193 let batch = child.new_child("2-1", None);
4194 let normal = batch.new_child("2-1-1", None);
4195 let flat = batch.new_flat_child("2-1-2", "parent", None);
4196 assert_eq!((level(&batch), level(&normal), level(&flat)), (2, 3, 2));
4197 for ctx in [&child, &batch, &normal, &flat] {
4198 assert_eq!(
4199 ctx.inner.preserve_child_depth, 5,
4200 "every constructor copies the depth"
4201 );
4202 }
4203
4204 for ctx in [
4205 DurableContext::test_context(),
4206 DurableContext::new_root(
4207 "arn".to_owned(),
4208 lambda_runtime::Context::default(),
4209 Arc::new(CheckpointLog::empty()),
4210 ),
4211 ] {
4212 assert_eq!((level(&ctx), ctx.inner.preserve_child_depth), (0, 0));
4213 }
4214 }
4215
4216 #[test]
4219 fn virtual_child_keeps_nesting_level() {
4220 let (root, _client) = DurableContext::test_context_with_depth(5);
4221 let level = |ctx: &DurableContext| ctx.inner.nesting_level;
4222 let under_root = root.new_virtual_child("1", None);
4223 let child = root.new_child("2", None);
4224 let under_child = child.new_virtual_child("2-1", None);
4225 assert_eq!((level(&under_root), level(&under_child)), (0, 1));
4226 for ctx in [&under_root, &under_child] {
4227 assert_eq!(ctx.inner.preserve_child_depth, 5);
4228 }
4229 assert_eq!(under_root.parent_wire_id(), None);
4232 assert_eq!(under_root.owners(), [] as [String; 0]);
4233 assert_eq!(under_child.parent_wire_id(), child.parent_wire_id());
4234 assert_eq!(under_child.owners(), child.owners());
4235 }
4236
4237 #[test]
4243 fn rebuilding_state_propagates_to_every_derived_context() {
4244 let root = DurableContext::test_context();
4245 let plain = root.new_child_rebuilding("1", None, false);
4246 let not_rebuilding = [
4247 root.clone(),
4248 root.new_child("2", None),
4249 root.new_virtual_child("3", None),
4250 root.new_flat_child("4", "parent", None),
4251 plain.new_child("1-1", None),
4252 plain.new_virtual_child("1-2", None),
4253 plain.new_flat_child("1-3", "parent", None),
4254 plain.clone(),
4255 ];
4256 for ctx in ¬_rebuilding {
4257 assert!(!ctx.is_rebuilding());
4258 }
4259
4260 let rebuilt = root.new_child_rebuilding("5", None, true);
4261 let virtual_child = rebuilt.new_virtual_child("5-1", None);
4262 let rebuilding = [
4263 rebuilt.new_child("5-2", None),
4264 rebuilt.new_child_rebuilding("5-3", None, false),
4265 rebuilt.new_flat_child("5-4", "parent", None),
4266 virtual_child.new_child("5-5", None),
4267 virtual_child.clone(),
4268 rebuilt,
4269 ];
4270 for ctx in &rebuilding {
4271 assert!(ctx.is_rebuilding());
4272 }
4273 }
4274
4275 #[test]
4278 fn preserves_children_table() {
4279 let depths = [0, 1, 2, u32::MAX];
4280 let row = |make: &dyn Fn(&DurableContext) -> DurableContext| -> Vec<bool> {
4281 depths
4282 .iter()
4283 .map(|&depth| {
4284 let (root, _client) = DurableContext::test_context_with_depth(depth);
4285 make(&root).preserves_children(1)
4286 })
4287 .collect()
4288 };
4289 assert_eq!(row(&|root| root.clone()), [false, true, true, true]);
4291 assert_eq!(
4293 row(&|root| root.new_child("1", None)),
4294 [false, false, true, true]
4295 );
4296 assert_eq!(
4298 row(&|root| root.new_child("1", None).new_child("1-1", None)),
4299 [false, false, false, true]
4300 );
4301 assert_eq!(
4303 row(&|root| root
4304 .new_child("1", None)
4305 .new_flat_child("1-1", "parent", None)),
4306 [false, false, true, true]
4307 );
4308 assert_eq!(
4310 row(&|root| root.new_child("1", None)),
4311 [false, false, true, true]
4312 );
4313 let (root, _client) = DurableContext::test_context_with_depth(u32::MAX);
4315 assert!(root.new_child("1", None).preserves_children(u32::MAX));
4316 }
4317
4318 #[test]
4321 fn combinator_methods_collect_without_claiming() {
4322 let ctx = DurableContext::new_root(
4323 "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
4324 lambda_runtime::Context::default(),
4325 Arc::new(CheckpointLog::empty()),
4326 );
4327 let inputs = || {
4330 let ctx = &ctx;
4331 (0..2).map(move |n| ctx.step(move |_| async move { Ok(n) }).future())
4332 };
4333 let try_join_all = ctx.try_join_all(inputs());
4334 let join_all = ctx.join_all(inputs());
4335 let select_ok = ctx.select_ok(inputs());
4336 let race = ctx.race(inputs());
4337 assert_eq!(
4338 ctx.mint_id().positional(),
4339 "9",
4340 "the eight steps claimed 1 to 8, the combinators nothing"
4341 );
4342 drop((try_join_all, join_all, select_ok, race));
4343 }
4344
4345 #[test]
4348 fn claim_without_active_body_records_nothing() {
4349 let ctx = DurableContext::test_context();
4350 let id = ctx.claim_operation_id("step");
4351 assert_eq!(id.positional(), "1");
4352 assert!(ctx.coordinator().fatal_error().is_none());
4353 }
4354
4355 #[test]
4359 fn claim_from_foreign_thread_records_fatal() {
4360 let ctx = DurableContext::test_context();
4361 assert_eq!(ctx.claim_operation_id("step").positional(), "1");
4362 assert!(ctx.coordinator().fatal_error().is_none());
4363
4364 let id = std::thread::scope(|scope| {
4365 scope
4366 .spawn(|| ctx.claim_operation_id("step"))
4367 .join()
4368 .expect("the claim does not panic")
4369 });
4370 assert_eq!(id.positional(), "2");
4371 let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4372 assert_eq!(fatal.error_type, "Error");
4373 assert_eq!(fatal.error_message, foreign_caller_message("step"));
4374 }
4375
4376 #[test]
4380 fn enforce_task_ownership_refuses_foreign_thread() {
4381 let ctx = DurableContext::test_context();
4382 assert!(ctx.enforce_task_ownership().is_ok());
4383 assert!(ctx.coordinator().fatal_error().is_none());
4384
4385 let result = std::thread::scope(|scope| {
4386 scope
4387 .spawn(|| ctx.enforce_task_ownership())
4388 .join()
4389 .expect("the check does not panic")
4390 });
4391 let err = result.expect_err("a foreign thread is refused");
4392 assert!(matches!(
4393 err.kind(),
4394 OperationErrorKind::Step(step) if matches!(step.kind(), StepErrorKind::ExecutionFailed)
4395 ));
4396 let source = std::error::Error::source(&err)
4399 .and_then(std::error::Error::source)
4400 .map(ToString::to_string);
4401 assert!(
4402 source
4403 .as_deref()
4404 .is_some_and(|msg| msg.contains("which does not own the context")),
4405 "unexpected source: {source:?}"
4406 );
4407 let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4408 assert_eq!(fatal.error_type, "Error");
4409 assert_eq!(fatal.error_message, foreign_caller_message("execute"));
4410 }
4411
4412 #[tokio::test]
4414 async fn claim_in_own_body_records_nothing() {
4415 let ctx = DurableContext::test_context();
4416 let id = run_body(ActiveBody::Context(String::new()), async {
4417 ctx.claim_operation_id("step")
4418 })
4419 .await;
4420 assert_eq!(id.positional(), "1");
4421 assert!(ctx.coordinator().fatal_error().is_none());
4422 }
4423
4424 #[tokio::test]
4429 async fn claim_in_own_body_on_raw_task_records_fatal() {
4430 let ctx = DurableContext::test_context();
4431 assert_eq!(ctx.claim_operation_id("step").positional(), "1");
4432 let foreign = ctx.clone();
4433 let id = tokio::spawn(run_body(ActiveBody::Context(String::new()), async move {
4434 foreign.claim_operation_id("step")
4435 }))
4436 .await
4437 .expect("the claim does not panic");
4438 assert_eq!(id.positional(), "2");
4439 let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4440 assert_eq!(fatal.error_type, "Error");
4441 assert_eq!(fatal.error_message, foreign_caller_message("step"));
4442 }
4443
4444 #[tokio::test]
4447 async fn claim_in_other_body_records_fatal() {
4448 let ctx = DurableContext::test_context();
4449 let id = run_body(ActiveBody::Context("3".to_owned()), async {
4450 ctx.claim_operation_id("wait")
4451 })
4452 .await;
4453 assert_eq!(id.positional(), "1");
4454 let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4455 assert_eq!(fatal.error_type, "Error");
4456 assert_eq!(
4457 fatal.error_message,
4458 "context usage error in \"wait\": the operation was called on context \"root\", \
4459 but the SDK is running the body of context \"3\". Inside run_in_child_context, \
4460 with_retry, map and parallel, call operations on the context passed to the closure."
4461 );
4462 }
4463
4464 #[tokio::test]
4466 async fn claim_in_step_body_records_fatal() {
4467 let ctx = DurableContext::test_context();
4468 let _id = run_body(ActiveBody::Step("4".to_owned()), async {
4469 ctx.claim_operation_id("map")
4470 })
4471 .await;
4472 let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4473 assert_eq!(fatal.error_type, "Error");
4474 assert_eq!(
4475 fatal.error_message,
4476 "context usage error in \"map\": the operation was called inside the body of \
4477 operation \"4\". A step body or a wait_for_condition check cannot call durable \
4478 operations; use run_in_child_context to group operations."
4479 );
4480 }
4481
4482 fn make_step_op(id: &str, result: &str) -> Operation {
4484 Operation::builder()
4485 .id(id)
4486 .r#type(aws_sdk_lambda::types::OperationType::Step)
4487 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
4488 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
4489 .step_details(
4490 aws_sdk_lambda::types::StepDetails::builder()
4491 .result(result)
4492 .build(),
4493 )
4494 .build()
4495 .unwrap()
4496 }
4497
4498 #[tokio::test]
4502 async fn checkpoint_updates_paginates_on_marker() {
4503 let all_ops = vec![
4505 make_step_op("step-1", "\"r1\""),
4506 make_step_op("step-2", "\"r2\""),
4507 make_step_op("step-3", "\"r3\""),
4508 ];
4509 let client = Arc::new(InMemoryExecutionClient::new(all_ops));
4510
4511 let page1_ops = vec![make_step_op("step-1", "\"r1\"")];
4514 client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
4515 page1_ops,
4516 "page-2-token".to_owned(),
4517 ));
4518
4519 let log = Arc::new(CheckpointLog::empty());
4520 let ctx = DurableContext::new_root_with_client(
4521 "arn:test".to_owned(),
4522 lambda_runtime::Context::default(),
4523 Arc::clone(&log),
4524 client.clone(),
4525 "initial-token".to_owned(),
4526 );
4527
4528 let result = ctx.checkpoint_updates(Vec::new()).await;
4530 assert!(result.is_ok());
4531
4532 assert!(log.get("step-1").is_some(), "step-1 must be in the log");
4534 assert!(
4535 log.get("step-2").is_some(),
4536 "step-2 must be in the log (paginated)"
4537 );
4538 assert!(
4539 log.get("step-3").is_some(),
4540 "step-3 must be in the log (paginated)"
4541 );
4542
4543 let get_state_count = *client.get_state_call_count.lock().unwrap();
4545 assert_eq!(
4546 get_state_count, 1,
4547 "get_state must be called for pagination"
4548 );
4549 }
4550
4551 #[derive(Debug)]
4556 struct PaginatedTokenValidatingClient {
4557 current_token: std::sync::Mutex<String>,
4558 counter: std::sync::atomic::AtomicU32,
4559 stale_checkpoint_tokens: std::sync::atomic::AtomicU32,
4560 stale_get_state_tokens: std::sync::atomic::AtomicU32,
4561 get_state_calls: std::sync::atomic::AtomicU32,
4562 }
4563
4564 impl PaginatedTokenValidatingClient {
4565 fn new(initial_token: &str) -> Self {
4566 Self {
4567 current_token: std::sync::Mutex::new(initial_token.to_owned()),
4568 counter: std::sync::atomic::AtomicU32::new(0),
4569 stale_checkpoint_tokens: std::sync::atomic::AtomicU32::new(0),
4570 stale_get_state_tokens: std::sync::atomic::AtomicU32::new(0),
4571 get_state_calls: std::sync::atomic::AtomicU32::new(0),
4572 }
4573 }
4574 }
4575
4576 impl ExecutionClient for PaginatedTokenValidatingClient {
4577 fn checkpoint(
4578 &self,
4579 _execution_arn: &str,
4580 checkpoint_token: &str,
4581 _updates: Vec<OperationUpdate>,
4582 ) -> std::pin::Pin<
4583 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
4584 > {
4585 let submitted = checkpoint_token.to_owned();
4586 Box::pin(async move {
4587 tokio::task::yield_now().await;
4589 let mut current = self
4590 .current_token
4591 .lock()
4592 .unwrap_or_else(std::sync::PoisonError::into_inner);
4593 if *current != submitted {
4594 self.stale_checkpoint_tokens
4595 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4596 return Err(ExecutionClientError::non_retryable(format!(
4597 "stale checkpoint token: expected {current}, got {submitted}"
4598 )));
4599 }
4600 let n = self
4601 .counter
4602 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4603 let next = format!("token-{}", n + 1);
4604 current.clone_from(&next);
4605 drop(current);
4606 Ok(CheckpointOutput {
4607 checkpoint_token: Some(next),
4608 updated_operations: Vec::new(),
4609 next_marker: Some("more-pages".to_owned()),
4612 })
4613 })
4614 }
4615
4616 fn get_state(
4617 &self,
4618 _execution_arn: &str,
4619 checkpoint_token: &str,
4620 _marker: Option<&str>,
4621 ) -> std::pin::Pin<
4622 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
4623 > {
4624 let submitted = checkpoint_token.to_owned();
4625 Box::pin(async move {
4626 self.get_state_calls
4627 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4628 tokio::task::yield_now().await;
4632 tokio::task::yield_now().await;
4633 let current = self
4634 .current_token
4635 .lock()
4636 .unwrap_or_else(std::sync::PoisonError::into_inner);
4637 if *current != submitted {
4638 self.stale_get_state_tokens
4639 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4640 }
4641 drop(current);
4642 Ok(GetStateOutput::new(Vec::new(), None))
4643 })
4644 }
4645 }
4646
4647 #[tokio::test]
4655 async fn concurrent_paginated_checkpoints_get_state_uses_current_token() {
4656 let client = Arc::new(PaginatedTokenValidatingClient::new("token-0"));
4657 let log = Arc::new(CheckpointLog::empty());
4658 let ctx = DurableContext::new_root_with_client(
4659 "arn:test".to_owned(),
4660 lambda_runtime::Context::default(),
4661 log,
4662 Arc::clone(&client) as Arc<dyn ExecutionClient>,
4663 "token-0".to_owned(),
4664 );
4665
4666 let (r1, r2, r3, r4) = tokio::join!(
4672 ctx.write_unchecked(Vec::new(), &[], None),
4673 ctx.write_unchecked(Vec::new(), &[], None),
4674 ctx.write_unchecked(Vec::new(), &[], None),
4675 ctx.write_unchecked(Vec::new(), &[], None),
4676 );
4677 assert!(r1.is_ok(), "checkpoint 1 failed: {r1:?}");
4678 assert!(r2.is_ok(), "checkpoint 2 failed: {r2:?}");
4679 assert!(r3.is_ok(), "checkpoint 3 failed: {r3:?}");
4680 assert!(r4.is_ok(), "checkpoint 4 failed: {r4:?}");
4681
4682 assert_eq!(
4683 client
4684 .get_state_calls
4685 .load(std::sync::atomic::Ordering::SeqCst),
4686 4,
4687 "every paginated checkpoint must trigger a get_state"
4688 );
4689 assert_eq!(
4690 client
4691 .stale_checkpoint_tokens
4692 .load(std::sync::atomic::Ordering::SeqCst),
4693 0,
4694 "no checkpoint may run with a stale token"
4695 );
4696 assert_eq!(
4697 client
4698 .stale_get_state_tokens
4699 .load(std::sync::atomic::Ordering::SeqCst),
4700 0,
4701 "the marker-triggered get_state must run with the current token \
4702 (token lock held through the paginated fetch)"
4703 );
4704 }
4705
4706 #[tokio::test]
4709 async fn checkpoint_updates_after_drive_uses_batch_limits() {
4710 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
4711 let ctx = DurableContext::new_root_with_client(
4712 "arn:test".to_owned(),
4713 lambda_runtime::Context::default(),
4714 Arc::new(CheckpointLog::empty()),
4715 Arc::clone(&client) as Arc<dyn ExecutionClient>,
4716 "token-0".to_owned(),
4717 );
4718 let ids: Vec<String> = (0..251).map(|n| format!("op-{n}")).collect();
4719 let updates = ids.iter().map(|id| make_update(id)).collect();
4720 ctx.checkpoint_updates_after_drive(updates)
4721 .await
4722 .expect("the write succeeds");
4723 assert_eq!(*client.checkpoint_call_count.lock().unwrap(), 2);
4724 let sent: Vec<String> = client
4725 .recorded_updates()
4726 .iter()
4727 .map(|update| update.id().to_owned())
4728 .collect();
4729 assert_eq!(sent, ids);
4730 }
4731
4732 #[tokio::test]
4735 async fn checkpoint_updates_no_pagination_when_no_marker() {
4736 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
4737 let log = Arc::new(CheckpointLog::empty());
4740 let ctx = DurableContext::new_root_with_client(
4741 "arn:test".to_owned(),
4742 lambda_runtime::Context::default(),
4743 log,
4744 client.clone(),
4745 "initial-token".to_owned(),
4746 );
4747
4748 let result = ctx.checkpoint_updates(Vec::new()).await;
4749 assert!(result.is_ok());
4750
4751 let get_state_count = *client.get_state_call_count.lock().unwrap();
4752 assert_eq!(
4753 get_state_count, 0,
4754 "get_state must NOT be called without a marker"
4755 );
4756 }
4757
4758 #[tokio::test]
4762 async fn bootstrap_pagination_fetches_all_pages() {
4763 let all_ops = vec![
4766 make_step_op("step-1", "\"result-1\""),
4767 make_step_op("step-2", "\"result-2\""),
4768 make_step_op("step-3", "\"result-3\""),
4769 ];
4770 let client: Arc<dyn ExecutionClient> = Arc::new(InMemoryExecutionClient::new(all_ops));
4771
4772 let full_state = client.get_state("arn:test", "token", None).await;
4774 assert!(full_state.is_ok());
4775 let full_state = full_state.unwrap();
4776
4777 let log = operations_to_checkpoint_log(&full_state.operations);
4778 assert!(log.get("step-1").is_some());
4779 assert!(log.get("step-2").is_some());
4780 assert!(log.get("step-3").is_some());
4781
4782 let r2 = log.get("step-2").unwrap();
4784 assert_eq!(r2.result.as_deref(), Some("\"result-2\""));
4785 assert_eq!(r2.status, CheckpointStatus::Succeeded);
4786 }
4787
4788 #[test]
4791 fn validate_replay_identity_passes_when_types_match() {
4792 let log = Arc::new(CheckpointLog::empty());
4793 let ctx = DurableContext::new_root(
4794 "arn:test".to_owned(),
4795 lambda_runtime::Context::default(),
4796 log,
4797 );
4798 let record = CheckpointRecord {
4799 id: "wire-1".to_owned(),
4800 status: CheckpointStatus::Succeeded,
4801 result: Some("42".to_owned()),
4802 error_type: None,
4803 error_message: None,
4804 error_data: None,
4805 stack_trace: None,
4806 attempt: 0,
4807 invoke_result: None,
4808 invoke_error_type: None,
4809 invoke_error_message: None,
4810 invoke_error_data: None,
4811 invoke_stack_trace: None,
4812 replay_children: false,
4813 callback_id: None,
4814 op_type: Some("Step".to_owned()),
4815 sub_type: Some("Step".to_owned()),
4816 due_at: None,
4817 op_name: Some("my-step".to_owned()),
4818 parent_id: None,
4819 start_timestamp: None,
4820 end_timestamp: None,
4821 };
4822
4823 let result =
4824 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("my-step"));
4825 assert!(result.is_ok());
4826 }
4827
4828 fn ctx_with_record_at_1(record: CheckpointRecord) -> DurableContext {
4833 let wire_id = crate::engine::compute_wire_id_public("1");
4834 let log = Arc::new(CheckpointLog::from_records(vec![(wire_id, record)]));
4835 DurableContext::new_root(
4836 "arn:test".to_owned(),
4837 lambda_runtime::Context::default(),
4838 log,
4839 )
4840 }
4841
4842 #[test]
4846 fn reject_record_at_virtual_child_messages() {
4847 let record = |op_type: Option<&str>| CheckpointRecord {
4848 id: "wire-1".to_owned(),
4849 status: CheckpointStatus::Succeeded,
4850 result: None,
4851 error_type: None,
4852 error_message: None,
4853 error_data: None,
4854 stack_trace: None,
4855 attempt: 0,
4856 invoke_result: None,
4857 invoke_error_type: None,
4858 invoke_error_message: None,
4859 invoke_error_data: None,
4860 invoke_stack_trace: None,
4861 replay_children: false,
4862 callback_id: None,
4863 op_type: op_type.map(str::to_owned),
4864 sub_type: Some("RunInChildContext".to_owned()),
4865 due_at: None,
4866 op_name: Some("slot".to_owned()),
4867 parent_id: None,
4868 start_timestamp: None,
4869 end_timestamp: None,
4870 };
4871 for (op_type, expected) in [
4872 (
4873 Some("Context"),
4874 "expected Context/RunInChildContext named \"slot\"",
4875 ),
4876 (None, "expected Unknown/RunInChildContext named \"slot\""),
4877 ] {
4878 let ctx = ctx_with_record_at_1(record(op_type));
4879 let op_id = ctx.mint_id();
4880 let err = ctx
4881 .reject_record_at_virtual_child(&op_id, "RunInChildContext", Some("slot"))
4882 .expect_err("a record at the slot");
4883 let message = crate::error::chain_string(&err);
4884 assert!(message.contains(expected), "{message}");
4885 assert!(
4886 message.contains("got virtual Context/RunInChildContext named \"slot\""),
4887 "{message}"
4888 );
4889 let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4890 assert_eq!(fatal.error_type, "NonDeterministicExecutionError");
4891 let next = ctx.mint_id();
4892 assert!(
4893 ctx.reject_record_at_virtual_child(&next, "RunInChildContext", None)
4894 .is_ok()
4895 );
4896 }
4897 }
4898
4899 #[test]
4900 fn checkpoint_view_validated_returns_view_on_identity_match() {
4901 let ctx = ctx_with_record_at_1(CheckpointRecord {
4902 id: "wire-1".to_owned(),
4903 status: CheckpointStatus::Succeeded,
4904 result: Some("42".to_owned()),
4905 error_type: None,
4906 error_message: None,
4907 error_data: None,
4908 stack_trace: None,
4909 attempt: 2,
4910 invoke_result: None,
4911 invoke_error_type: None,
4912 invoke_error_message: None,
4913 invoke_error_data: None,
4914 invoke_stack_trace: None,
4915 replay_children: true,
4916 callback_id: None,
4917 op_type: Some("Step".to_owned()),
4918 sub_type: Some("Step".to_owned()),
4919 due_at: None,
4920 op_name: Some("my-step".to_owned()),
4921 parent_id: None,
4922 start_timestamp: None,
4923 end_timestamp: None,
4924 });
4925
4926 let result =
4927 ctx.checkpoint_view_validated("1", "wire-1", "Step", Some("Step"), Some("my-step"));
4928 assert!(result.is_ok());
4929 let view = result.unwrap();
4930 assert!(view.is_some());
4931 let view = view.unwrap();
4932 assert_eq!(view.status, CheckpointStatus::Succeeded);
4933 assert_eq!(view.attempt, 2);
4934 assert!(view.replay_children);
4935
4936 let live = ctx.checkpoint_view_validated("2", "wire-2", "Step", Some("Step"), None);
4938 assert!(matches!(live, Ok(None)));
4939 }
4940
4941 #[test]
4942 fn checkpoint_view_validated_errors_on_identity_mismatch() {
4943 let ctx = ctx_with_record_at_1(CheckpointRecord {
4944 id: "wire-1".to_owned(),
4945 status: CheckpointStatus::Succeeded,
4946 result: None,
4947 error_type: None,
4948 error_message: None,
4949 error_data: None,
4950 stack_trace: None,
4951 attempt: 0,
4952 invoke_result: None,
4953 invoke_error_type: None,
4954 invoke_error_message: None,
4955 invoke_error_data: None,
4956 invoke_stack_trace: None,
4957 replay_children: false,
4958 callback_id: None,
4959 op_type: Some("Step".to_owned()),
4960 sub_type: Some("Step".to_owned()),
4961 due_at: None,
4962 op_name: None,
4963 parent_id: None,
4964 start_timestamp: None,
4965 end_timestamp: None,
4966 });
4967
4968 let result = ctx.checkpoint_view_validated("1", "wire-1", "Wait", Some("Wait"), None);
4971 assert!(result.is_err());
4972 let err = result.unwrap_err();
4973 assert!(matches!(
4974 err.kind(),
4975 OperationErrorKind::NonDeterministicExecution(_)
4976 ));
4977 let display = format!("{err:#}");
4978 assert!(
4979 display.contains("Step") && display.contains("Wait"),
4980 "expected both identities in error: {display}"
4981 );
4982 }
4983
4984 #[test]
4988 #[expect(clippy::panic)] fn checkpoint_view_validated_fails_execution_on_unrecognized_status() {
4990 let ctx = ctx_with_record_at_1(CheckpointRecord {
4991 id: "wire-1".to_owned(),
4992 status: CheckpointStatus::Unknown("PAUSED".to_owned()),
4993 result: None,
4994 error_type: None,
4995 error_message: None,
4996 error_data: None,
4997 stack_trace: None,
4998 attempt: 0,
4999 invoke_result: None,
5000 invoke_error_type: None,
5001 invoke_error_message: None,
5002 invoke_error_data: None,
5003 invoke_stack_trace: None,
5004 replay_children: false,
5005 callback_id: None,
5006 op_type: Some("Step".to_owned()),
5007 sub_type: Some("Step".to_owned()),
5008 due_at: None,
5009 op_name: None,
5010 parent_id: None,
5011 start_timestamp: None,
5012 end_timestamp: None,
5013 });
5014
5015 assert!(
5016 ctx.coordinator().fatal_error().is_none(),
5017 "no fatal recorded before the replay check"
5018 );
5019
5020 let result = ctx.checkpoint_view_validated("1", "wire-1", "Step", Some("Step"), None);
5022 assert!(result.is_err());
5023 let err = result.unwrap_err();
5024 let OperationErrorKind::NonDeterministicExecution(nde) = err.kind() else {
5025 panic!("expected NonDeterministicExecution, got {:?}", err.kind());
5026 };
5027 let NonDeterministicExecutionErrorKind::UnrecognizedStatus(details) = nde.kind() else {
5028 panic!("expected UnrecognizedStatus, got {:?}", nde.kind());
5029 };
5030 assert_eq!(details.wire_id(), "wire-1");
5031 assert_eq!(details.status(), "PAUSED");
5032 let display = format!("{err:#}");
5033 assert!(
5034 display.contains("PAUSED"),
5035 "error must name the raw status: {display}"
5036 );
5037
5038 let fatal = ctx
5041 .coordinator()
5042 .fatal_error()
5043 .expect("unrecognized status must record a fatal error");
5044 assert_eq!(fatal.error_type, "NonDeterministicExecutionError");
5045 assert!(
5046 fatal.error_message.contains("PAUSED"),
5047 "fatal message must carry the raw status: {}",
5048 fatal.error_message
5049 );
5050 }
5051
5052 #[test]
5055 fn checkpoint_view_validated_known_status_records_no_fatal() {
5056 let ctx = ctx_with_record_at_1(CheckpointRecord {
5057 id: "wire-1".to_owned(),
5058 status: CheckpointStatus::TimedOut,
5059 result: None,
5060 error_type: None,
5061 error_message: None,
5062 error_data: None,
5063 stack_trace: None,
5064 attempt: 0,
5065 invoke_result: None,
5066 invoke_error_type: None,
5067 invoke_error_message: None,
5068 invoke_error_data: None,
5069 invoke_stack_trace: None,
5070 replay_children: false,
5071 callback_id: None,
5072 op_type: Some("Step".to_owned()),
5073 sub_type: Some("Step".to_owned()),
5074 due_at: None,
5075 op_name: None,
5076 parent_id: None,
5077 start_timestamp: None,
5078 end_timestamp: None,
5079 });
5080
5081 let result = ctx.checkpoint_view_validated("1", "wire-1", "Step", Some("Step"), None);
5082 assert!(matches!(result, Ok(Some(_))));
5083 assert!(ctx.coordinator().fatal_error().is_none());
5084 }
5085
5086 #[test]
5087 fn targeted_getters_return_single_fields() {
5088 let ctx = ctx_with_record_at_1(CheckpointRecord {
5089 id: "wire-1".to_owned(),
5090 status: CheckpointStatus::Failed,
5091 result: Some(r#""state""#.to_owned()),
5092 error_type: Some("SomeError".to_owned()),
5093 error_message: Some("it broke".to_owned()),
5094 error_data: None,
5095 stack_trace: None,
5096 attempt: 5,
5097 invoke_result: None,
5098 invoke_error_type: None,
5099 invoke_error_message: None,
5100 invoke_error_data: None,
5101 invoke_stack_trace: None,
5102 replay_children: false,
5103 callback_id: Some("cb-42".to_owned()),
5104 op_type: None,
5105 sub_type: None,
5106 due_at: None,
5107 op_name: None,
5108 parent_id: None,
5109 start_timestamp: None,
5110 end_timestamp: None,
5111 });
5112
5113 assert!(ctx.has_checkpoint_record("1"));
5114 assert!(!ctx.has_checkpoint_record("2"));
5115
5116 assert_eq!(
5117 ctx.checkpoint_result_payload("1").as_deref(),
5118 Some(r#""state""#)
5119 );
5120 assert!(ctx.checkpoint_result_payload("2").is_none());
5121
5122 assert_eq!(ctx.checkpoint_callback_id("1").as_deref(), Some("cb-42"));
5123
5124 let wire = ctx.checkpoint_wire_error("1");
5125 let wire = wire.expect("record 1 exists");
5126 assert_eq!(wire.error_type(), Some("SomeError"));
5127 assert_eq!(wire.error_message(), Some("it broke"));
5128 assert!(ctx.checkpoint_wire_error("2").is_none());
5129
5130 let view = ctx.checkpoint_status_view("1");
5131 assert!(view.is_some());
5132 let view = view.unwrap();
5133 assert_eq!(view.status, CheckpointStatus::Failed);
5134 assert_eq!(view.attempt, 5);
5135 }
5136
5137 #[test]
5138 fn checkpoint_terminal_replay_projects_payload_and_error_fields() {
5139 let ctx = ctx_with_record_at_1(CheckpointRecord {
5140 id: "wire-1".to_owned(),
5141 status: CheckpointStatus::Failed,
5142 result: Some(r#""summary""#.to_owned()),
5143 error_type: Some("BatchError".to_owned()),
5144 error_message: Some("batch broke".to_owned()),
5145 error_data: Some(r#"{"code":7}"#.to_owned()),
5146 stack_trace: Some(vec!["frame-a".to_owned()]),
5147 attempt: 3,
5148 invoke_result: Some("never-projected".to_owned()),
5149 invoke_error_type: Some("never-projected".to_owned()),
5150 invoke_error_message: Some("never-projected".to_owned()),
5151 invoke_error_data: None,
5152 invoke_stack_trace: None,
5153 replay_children: true,
5154 callback_id: Some("cb-99".to_owned()),
5155 op_type: Some("Context".to_owned()),
5156 sub_type: Some("Map".to_owned()),
5157 due_at: None,
5158 op_name: Some("batch".to_owned()),
5159 parent_id: None,
5160 start_timestamp: None,
5161 end_timestamp: None,
5162 });
5163
5164 let snapshot = ctx.checkpoint_terminal_replay("1");
5165 assert_eq!(
5166 snapshot,
5167 Some(TerminalReplaySnapshot {
5168 status: CheckpointStatus::Failed,
5169 replay_children: true,
5170 result: Some(r#""summary""#.to_owned()),
5171 error_message: Some("batch broke".to_owned()),
5172 error_type: Some("BatchError".to_owned()),
5173 error_data: Some(r#"{"code":7}"#.to_owned()),
5174 stack_trace: Some(vec!["frame-a".to_owned()]),
5175 })
5176 );
5177
5178 assert!(ctx.checkpoint_terminal_replay("2").is_none());
5180 }
5181
5182 #[test]
5183 fn validate_replay_identity_fails_on_type_mismatch() {
5184 let log = Arc::new(CheckpointLog::empty());
5185 let ctx = DurableContext::new_root(
5186 "arn:test".to_owned(),
5187 lambda_runtime::Context::default(),
5188 log,
5189 );
5190 let record = CheckpointRecord {
5191 id: "wire-1".to_owned(),
5192 status: CheckpointStatus::Succeeded,
5193 result: None,
5194 error_type: None,
5195 error_message: None,
5196 error_data: None,
5197 stack_trace: None,
5198 attempt: 0,
5199 invoke_result: None,
5200 invoke_error_type: None,
5201 invoke_error_message: None,
5202 invoke_error_data: None,
5203 invoke_stack_trace: None,
5204 replay_children: false,
5205 callback_id: None,
5206 op_type: Some("Step".to_owned()),
5207 sub_type: Some("Step".to_owned()),
5208 due_at: None,
5209 op_name: None,
5210 parent_id: None,
5211 start_timestamp: None,
5212 end_timestamp: None,
5213 };
5214
5215 let result = ctx.validate_replay_identity(&record, "wire-1", "Wait", Some("Wait"), None);
5217 assert!(result.is_err());
5218 let err = result.unwrap_err();
5219 assert!(matches!(
5220 err.kind(),
5221 OperationErrorKind::NonDeterministicExecution(_)
5222 ));
5223 let display = format!("{err:#}");
5224 assert!(
5225 display.contains("Step"),
5226 "expected Step in error: {display}"
5227 );
5228 assert!(
5229 display.contains("Wait"),
5230 "expected Wait in error: {display}"
5231 );
5232 }
5233
5234 #[test]
5235 fn validate_replay_identity_fails_on_subtype_mismatch() {
5236 let log = Arc::new(CheckpointLog::empty());
5237 let ctx = DurableContext::new_root(
5238 "arn:test".to_owned(),
5239 lambda_runtime::Context::default(),
5240 log,
5241 );
5242 let record = CheckpointRecord {
5243 id: "wire-1".to_owned(),
5244 status: CheckpointStatus::Succeeded,
5245 result: None,
5246 error_type: None,
5247 error_message: None,
5248 error_data: None,
5249 stack_trace: None,
5250 attempt: 0,
5251 invoke_result: None,
5252 invoke_error_type: None,
5253 invoke_error_message: None,
5254 invoke_error_data: None,
5255 invoke_stack_trace: None,
5256 replay_children: false,
5257 callback_id: None,
5258 op_type: Some("Step".to_owned()),
5259 sub_type: Some("Step".to_owned()),
5260 due_at: None,
5261 op_name: None,
5262 parent_id: None,
5263 start_timestamp: None,
5264 end_timestamp: None,
5265 };
5266
5267 let result =
5269 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("WaitForCondition"), None);
5270 assert!(result.is_err());
5271 let err = result.unwrap_err();
5272 assert!(matches!(
5273 err.kind(),
5274 OperationErrorKind::NonDeterministicExecution(_)
5275 ));
5276 }
5277
5278 #[test]
5279 fn validate_replay_identity_fails_on_name_mismatch() {
5280 let log = Arc::new(CheckpointLog::empty());
5281 let ctx = DurableContext::new_root(
5282 "arn:test".to_owned(),
5283 lambda_runtime::Context::default(),
5284 log,
5285 );
5286 let record = CheckpointRecord {
5287 id: "wire-1".to_owned(),
5288 status: CheckpointStatus::Succeeded,
5289 result: None,
5290 error_type: None,
5291 error_message: None,
5292 error_data: None,
5293 stack_trace: None,
5294 attempt: 0,
5295 invoke_result: None,
5296 invoke_error_type: None,
5297 invoke_error_message: None,
5298 invoke_error_data: None,
5299 invoke_stack_trace: None,
5300 replay_children: false,
5301 callback_id: None,
5302 op_type: Some("Step".to_owned()),
5303 sub_type: Some("Step".to_owned()),
5304 due_at: None,
5305 op_name: Some("fetch-user".to_owned()),
5306 parent_id: None,
5307 start_timestamp: None,
5308 end_timestamp: None,
5309 };
5310
5311 let result = ctx.validate_replay_identity(
5313 &record,
5314 "wire-1",
5315 "Step",
5316 Some("Step"),
5317 Some("fetch-order"),
5318 );
5319 assert!(result.is_err());
5320 let err = result.unwrap_err();
5321 let display = format!("{err:#}");
5322 assert!(
5323 display.contains("fetch-user"),
5324 "expected name in error: {display}"
5325 );
5326 assert!(
5327 display.contains("fetch-order"),
5328 "expected claimed name in error: {display}"
5329 );
5330 }
5331
5332 #[test]
5333 fn validate_replay_identity_skips_when_no_identity_stored() {
5334 let log = Arc::new(CheckpointLog::empty());
5335 let ctx = DurableContext::new_root(
5336 "arn:test".to_owned(),
5337 lambda_runtime::Context::default(),
5338 log,
5339 );
5340 let record = CheckpointRecord {
5341 id: "wire-1".to_owned(),
5342 status: CheckpointStatus::Succeeded,
5343 result: Some("1".to_owned()),
5344 error_type: None,
5345 error_message: None,
5346 error_data: None,
5347 stack_trace: None,
5348 attempt: 0,
5349 invoke_result: None,
5350 invoke_error_type: None,
5351 invoke_error_message: None,
5352 invoke_error_data: None,
5353 invoke_stack_trace: None,
5354 replay_children: false,
5355 callback_id: None,
5356 op_type: None,
5357 sub_type: None,
5358 due_at: None,
5359 op_name: None,
5360 parent_id: None,
5361 start_timestamp: None,
5362 end_timestamp: None,
5363 };
5364
5365 let result = ctx.validate_replay_identity(&record, "wire-1", "Wait", Some("Wait"), None);
5368 assert!(result.is_ok());
5369 }
5370
5371 #[test]
5372 fn validate_replay_identity_case_insensitive_type_match() {
5373 let log = Arc::new(CheckpointLog::empty());
5374 let ctx = DurableContext::new_root(
5375 "arn:test".to_owned(),
5376 lambda_runtime::Context::default(),
5377 log,
5378 );
5379 let record = CheckpointRecord {
5380 id: "wire-1".to_owned(),
5381 status: CheckpointStatus::Succeeded,
5382 result: None,
5383 error_type: None,
5384 error_message: None,
5385 error_data: None,
5386 stack_trace: None,
5387 attempt: 0,
5388 invoke_result: None,
5389 invoke_error_type: None,
5390 invoke_error_message: None,
5391 invoke_error_data: None,
5392 invoke_stack_trace: None,
5393 replay_children: false,
5394 callback_id: None,
5395 op_type: Some("STEP".to_owned()),
5396 sub_type: Some("STEP".to_owned()),
5397 due_at: None,
5398 op_name: None,
5399 parent_id: None,
5400 start_timestamp: None,
5401 end_timestamp: None,
5402 };
5403
5404 let result = ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), None);
5406 assert!(result.is_ok());
5407 }
5408
5409 #[test]
5415 fn validate_replay_identity_inline_chained_invoke_replay_matches() {
5416 let log = Arc::new(CheckpointLog::empty());
5417 let ctx = DurableContext::new_root(
5418 "arn:test".to_owned(),
5419 lambda_runtime::Context::default(),
5420 log,
5421 );
5422 let record = CheckpointRecord {
5423 id: "wire-1".to_owned(),
5424 status: CheckpointStatus::Succeeded,
5425 result: None,
5426 error_type: None,
5427 error_message: None,
5428 error_data: None,
5429 stack_trace: None,
5430 attempt: 0,
5431 invoke_result: Some("\"ok\"".to_owned()),
5432 invoke_error_type: None,
5433 invoke_error_message: None,
5434 invoke_error_data: None,
5435 invoke_stack_trace: None,
5436 replay_children: false,
5437 callback_id: None,
5438 op_type: Some("CHAINED_INVOKE".to_owned()),
5441 sub_type: Some("ChainedInvoke".to_owned()),
5442 due_at: None,
5443 op_name: None,
5444 parent_id: None,
5445 start_timestamp: None,
5446 end_timestamp: None,
5447 };
5448
5449 let result = ctx.validate_replay_identity(
5450 &record,
5451 "wire-1",
5452 "ChainedInvoke",
5453 Some("ChainedInvoke"),
5454 None,
5455 );
5456 assert!(
5457 result.is_ok(),
5458 "inline CHAINED_INVOKE record must match claimed ChainedInvoke: {result:?}"
5459 );
5460 }
5461
5462 #[test]
5465 fn validate_replay_identity_canonicalized_types_still_mismatch() {
5466 let log = Arc::new(CheckpointLog::empty());
5467 let ctx = DurableContext::new_root(
5468 "arn:test".to_owned(),
5469 lambda_runtime::Context::default(),
5470 log,
5471 );
5472 let record = CheckpointRecord {
5473 id: "wire-1".to_owned(),
5474 status: CheckpointStatus::Succeeded,
5475 result: None,
5476 error_type: None,
5477 error_message: None,
5478 error_data: None,
5479 stack_trace: None,
5480 attempt: 0,
5481 invoke_result: None,
5482 invoke_error_type: None,
5483 invoke_error_message: None,
5484 invoke_error_data: None,
5485 invoke_stack_trace: None,
5486 replay_children: false,
5487 callback_id: None,
5488 op_type: Some("CHAINED_INVOKE".to_owned()),
5489 sub_type: Some("ChainedInvoke".to_owned()),
5490 due_at: None,
5491 op_name: None,
5492 parent_id: None,
5493 start_timestamp: None,
5494 end_timestamp: None,
5495 };
5496
5497 let result =
5498 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("ChainedInvoke"), None);
5499 assert!(
5500 result.is_err(),
5501 "Step claimed against CHAINED_INVOKE record must mismatch"
5502 );
5503 }
5504
5505 #[test]
5508 fn canonical_op_type_bridges_wire_and_pascal_spellings() {
5509 for (pascal, wire) in [
5510 ("Callback", "CALLBACK"),
5511 ("ChainedInvoke", "CHAINED_INVOKE"),
5512 ("Context", "CONTEXT"),
5513 ("Execution", "EXECUTION"),
5514 ("Step", "STEP"),
5515 ("Wait", "WAIT"),
5516 ] {
5517 assert_eq!(
5518 canonical_op_type(pascal),
5519 canonical_op_type(wire),
5520 "{pascal} and {wire} must canonicalize identically"
5521 );
5522 }
5523 assert_ne!(
5524 canonical_op_type("Step"),
5525 canonical_op_type("ChainedInvoke"),
5526 "distinct types must stay distinct"
5527 );
5528 }
5529
5530 fn record_with_identity(
5533 op_type: Option<&str>,
5534 sub_type: Option<&str>,
5535 op_name: Option<&str>,
5536 ) -> CheckpointRecord {
5537 CheckpointRecord {
5538 id: "wire-1".to_owned(),
5539 status: CheckpointStatus::Succeeded,
5540 result: None,
5541 error_type: None,
5542 error_message: None,
5543 error_data: None,
5544 stack_trace: None,
5545 attempt: 0,
5546 invoke_result: None,
5547 invoke_error_type: None,
5548 invoke_error_message: None,
5549 invoke_error_data: None,
5550 invoke_stack_trace: None,
5551 replay_children: false,
5552 callback_id: None,
5553 op_type: op_type.map(ToOwned::to_owned),
5554 sub_type: sub_type.map(ToOwned::to_owned),
5555 due_at: None,
5556 op_name: op_name.map(ToOwned::to_owned),
5557 parent_id: None,
5558 start_timestamp: None,
5559 end_timestamp: None,
5560 }
5561 }
5562
5563 fn test_ctx() -> DurableContext {
5564 DurableContext::new_root(
5565 "arn:test".to_owned(),
5566 lambda_runtime::Context::default(),
5567 Arc::new(CheckpointLog::empty()),
5568 )
5569 }
5570
5571 #[test]
5572 fn validate_replay_identity_fails_when_stored_name_removed() {
5573 let ctx = test_ctx();
5578 let record = record_with_identity(Some("Step"), Some("Step"), Some("my-step"));
5579
5580 let result = ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), None);
5581 let err = result.unwrap_err();
5582 assert!(matches!(
5583 err.kind(),
5584 OperationErrorKind::NonDeterministicExecution(_)
5585 ));
5586 let display = format!("{err:#}");
5587 assert!(
5588 display.contains("my-step"),
5589 "expected stored name in error: {display}"
5590 );
5591 }
5592
5593 #[test]
5594 fn validate_replay_identity_fails_when_name_added() {
5595 let ctx = test_ctx();
5598 let record = record_with_identity(Some("Step"), Some("Step"), None);
5599
5600 let result =
5601 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("new-name"));
5602 let err = result.unwrap_err();
5603 assert!(matches!(
5604 err.kind(),
5605 OperationErrorKind::NonDeterministicExecution(_)
5606 ));
5607 let display = format!("{err:#}");
5608 assert!(
5609 display.contains("new-name"),
5610 "expected claimed name in error: {display}"
5611 );
5612 }
5613
5614 #[test]
5615 fn validate_replay_identity_fails_when_subtype_dropped() {
5616 let ctx = test_ctx();
5618 let record = record_with_identity(Some("Context"), Some("Map"), None);
5619
5620 let result = ctx.validate_replay_identity(&record, "wire-1", "Context", None, None);
5621 assert!(
5622 result.is_err(),
5623 "expected mismatch when stored sub-type is dropped from the claim"
5624 );
5625
5626 let record = record_with_identity(Some("Context"), None, None);
5628 let result = ctx.validate_replay_identity(&record, "wire-1", "Context", Some("Map"), None);
5629 assert!(
5630 result.is_err(),
5631 "expected mismatch when a sub-type is added to the claim"
5632 );
5633 }
5634
5635 #[test]
5636 fn validate_replay_identity_untyped_record_skips_name_and_subtype() {
5637 let ctx = test_ctx();
5640 let record = record_with_identity(None, None, None);
5641
5642 let result =
5643 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("any-name"));
5644 assert!(result.is_ok());
5645 }
5646
5647 #[test]
5648 fn validate_replay_identity_empty_claimed_name_matches_stored_none() {
5649 let ctx = test_ctx();
5652 let record = record_with_identity(Some("Context"), Some("MapIteration"), None);
5653
5654 let result = ctx.validate_replay_identity(
5655 &record,
5656 "wire-1",
5657 "Context",
5658 Some("MapIteration"),
5659 Some(""),
5660 );
5661 assert!(
5662 result.is_ok(),
5663 "claimed empty name must match a stored None: {result:?}"
5664 );
5665 }
5666
5667 #[test]
5668 fn validate_replay_identity_empty_stored_name_matches_claimed_none() {
5669 let ctx = test_ctx();
5672 let record = record_with_identity(Some("Step"), Some("Step"), Some(""));
5673
5674 let result = ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), None);
5675 assert!(
5676 result.is_ok(),
5677 "stored empty name must match a claimed None: {result:?}"
5678 );
5679 }
5680
5681 #[test]
5682 fn validate_replay_identity_empty_name_still_mismatches_real_name() {
5683 let ctx = test_ctx();
5686 let record = record_with_identity(Some("Step"), Some("Step"), Some("real"));
5687 let result =
5688 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some(""));
5689 assert!(
5690 result.is_err(),
5691 "empty claim vs stored 'real' must mismatch"
5692 );
5693
5694 let record = record_with_identity(Some("Step"), Some("Step"), None);
5695 let result =
5696 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("real"));
5697 assert!(
5698 result.is_err(),
5699 "claimed 'real' vs stored None must mismatch"
5700 );
5701 }
5702
5703 #[test]
5704 fn validate_replay_identity_mismatch_records_fatal_error() {
5705 let ctx = test_ctx();
5709 let record = record_with_identity(Some("Step"), Some("Step"), Some("alpha"));
5710
5711 assert!(
5712 ctx.coordinator().fatal_error().is_none(),
5713 "no fatal recorded before validation"
5714 );
5715 let result =
5716 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("beta"));
5717 assert!(result.is_err());
5718
5719 let fatal = ctx
5720 .coordinator()
5721 .fatal_error()
5722 .expect("mismatch must record a fatal error");
5723 assert_eq!(fatal.error_type, "NonDeterministicExecutionError");
5724 assert!(
5725 fatal.error_message.contains("alpha") && fatal.error_message.contains("beta"),
5726 "fatal message must name both identities: {}",
5727 fatal.error_message
5728 );
5729 }
5730
5731 #[test]
5732 fn validate_replay_identity_pass_records_no_fatal() {
5733 let ctx = test_ctx();
5734 let record = record_with_identity(Some("Step"), Some("Step"), Some("same"));
5735
5736 let result =
5737 ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("same"));
5738 assert!(result.is_ok());
5739 assert!(
5740 ctx.coordinator().fatal_error().is_none(),
5741 "a passing validation must not record a fatal error"
5742 );
5743 }
5744
5745 #[test]
5748 fn debug_output_redacts_checkpoint_token() {
5749 let secret_token = "super-secret-credential-value-12345";
5750 let log = Arc::new(CheckpointLog::empty());
5751 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5752 let ctx = DurableContext::new_root_with_client(
5753 "arn:aws:lambda:us-east-1:123456789012:function:my-fn".to_owned(),
5754 lambda_runtime::Context::default(),
5755 log,
5756 client,
5757 secret_token.to_owned(),
5758 );
5759
5760 let debug_output = format!("{ctx:?}");
5761
5762 assert!(
5764 !debug_output.contains(secret_token),
5765 "checkpoint_token value leaked in Debug output: {debug_output}"
5766 );
5767
5768 assert!(
5770 debug_output.contains("<redacted>"),
5771 "expected '<redacted>' in Debug output: {debug_output}"
5772 );
5773
5774 assert!(
5776 debug_output.contains("arn:aws:lambda:us-east-1:123456789012:function:my-fn"),
5777 "expected execution_arn in Debug output: {debug_output}"
5778 );
5779 }
5780
5781 #[test]
5785 fn derived_contexts_share_the_root_plugins() {
5786 let plugins = crate::plugin::test_support::invocation_plugins();
5787 let root = DurableContext::new_root_with_client_and_defaults(
5788 "arn:test".to_owned(),
5789 lambda_runtime::Context::default(),
5790 Arc::new(CheckpointLog::empty()),
5791 Arc::new(InMemoryExecutionClient::new(Vec::new())),
5792 "token-0".to_owned(),
5793 Duration::ZERO,
5794 0,
5795 Some(Arc::clone(&plugins)),
5796 );
5797 let same = |ctx: &DurableContext| {
5798 ctx.plugins()
5799 .is_some_and(|shared| std::ptr::eq(shared, Arc::as_ptr(&plugins)))
5800 };
5801 assert!(same(&root), "the root holds the plugins");
5802 let child = root.new_child("1", None);
5803 assert!(same(&child), "new_child copies the plugins");
5804 assert!(
5805 same(&child.new_child("1-1", None)),
5806 "a grandchild shares them"
5807 );
5808 assert!(
5809 same(&root.new_flat_child("2", "parent", None)),
5810 "new_flat_child copies the plugins"
5811 );
5812 assert!(
5813 DurableContext::test_context().plugins().is_none(),
5814 "a root constructor without the parameter sets none"
5815 );
5816 }
5817
5818 fn coalescing_ctx(client: Arc<dyn ExecutionClient>, delay: Duration) -> DurableContext {
5823 DurableContext::new_root_with_client_and_defaults(
5824 "arn:test".to_owned(),
5825 lambda_runtime::Context::default(),
5826 Arc::new(CheckpointLog::empty()),
5827 client,
5828 "token-0".to_owned(),
5829 delay,
5830 0,
5831 None,
5832 )
5833 }
5834
5835 fn make_update(id: &str) -> OperationUpdate {
5837 OperationUpdate::builder()
5838 .id(id.to_owned())
5839 .r#type(aws_sdk_lambda::types::OperationType::Step)
5840 .action(OperationAction::Start)
5841 .build()
5842 .expect("all required OperationUpdate fields set")
5843 }
5844
5845 #[test]
5849 fn batch_flush_without_runtime_returns_invocation_error() {
5850 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5851 let ctx = coalescing_ctx(client.clone(), Duration::ZERO);
5852 let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
5853 ctx.checkpoint_updates(vec![make_update("op-a")]).await
5854 });
5855 let std::task::Poll::Ready(Err(err)) = polled else {
5856 unreachable!("expected an immediate error, got {polled:?}");
5857 };
5858 assert_eq!(err.scope(), ExecutionClientErrorScope::Invocation);
5859 assert!(
5860 err.to_string().contains(crate::task::NO_RUNTIME_MESSAGE),
5861 "unexpected error: {err}"
5862 );
5863 assert!(client.recorded_updates().is_empty(), "nothing is written");
5864 }
5865
5866 #[test]
5870 fn unspawned_flush_of_a_claimed_batch_publishes_nothing() {
5871 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5872 let ctx = coalescing_ctx(client.clone(), Duration::from_secs(60));
5873 let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
5874 let coalescer = Arc::clone(&ctx.inner.coalescer);
5875 let (batch, _) = coalescer
5876 .join(
5877 ctx.admission(),
5878 ctx.coordinator(),
5879 ctx.owners(),
5880 vec![TrackedUpdate::capture(make_update("op-a"), 1)],
5881 )
5882 .expect("admitted");
5883 let claimed = coalescer.take_batch(&batch);
5884 ctx.spawn_batch_flush(&coalescer, &batch, Duration::ZERO);
5885 (claimed.is_some(), batch.result_clone().is_none())
5886 });
5887 assert!(
5888 matches!(polled, std::task::Poll::Ready((true, true))),
5889 "{polled:?}"
5890 );
5891 assert!(client.recorded_updates().is_empty(), "nothing is written");
5892 }
5893
5894 #[tokio::test(start_paused = true)]
5897 async fn coalesced_checkpoints_batch_into_one_call() {
5898 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5899 let ctx = coalescing_ctx(client.clone(), Duration::from_millis(50));
5900
5901 let (a, b) = tokio::join!(
5902 ctx.checkpoint_updates(vec![make_update("op-a")]),
5903 ctx.checkpoint_updates(vec![make_update("op-b")]),
5904 );
5905 assert!(a.is_ok(), "first coalesced caller succeeds");
5906 assert!(b.is_ok(), "second coalesced caller succeeds");
5907
5908 let call_count = *client.checkpoint_call_count.lock().unwrap();
5909 assert_eq!(call_count, 1, "both updates must share one API call");
5910
5911 let ids: Vec<String> = client
5912 .recorded_updates()
5913 .iter()
5914 .map(|u| u.id.clone())
5915 .collect();
5916 assert_eq!(
5917 ids,
5918 vec!["op-a".to_owned(), "op-b".to_owned()],
5919 "the single call carries both updates in join order"
5920 );
5921 }
5922
5923 #[tokio::test(start_paused = true)]
5927 async fn urgent_checkpoint_flushes_without_waiting_the_window() {
5928 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5929 let ctx = coalescing_ctx(client.clone(), Duration::from_hours(1));
5932
5933 let start = tokio::time::Instant::now();
5934 ctx.submit_checkpoint_updates_urgent(vec![make_update("op-urgent")])
5935 .await
5936 .expect("urgent checkpoint succeeds");
5937
5938 assert_eq!(
5939 start.elapsed(),
5940 Duration::ZERO,
5941 "urgent flush must not wait for the coalescing window"
5942 );
5943 let call_count = *client.checkpoint_call_count.lock().unwrap();
5944 assert_eq!(call_count, 1);
5945 }
5946
5947 #[tokio::test(start_paused = true)]
5951 async fn flush_pending_checkpoints_drains_the_buffer() {
5952 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5953 let ctx = coalescing_ctx(client.clone(), Duration::from_hours(1));
5954
5955 let start = tokio::time::Instant::now();
5956 let buffered = tokio::spawn({
5957 let ctx = ctx.clone();
5958 async move {
5959 ctx.checkpoint_updates(vec![make_update("op-buffered")])
5960 .await
5961 }
5962 });
5963 tokio::time::sleep(Duration::from_millis(1)).await;
5965
5966 ctx.flush_pending_checkpoints().await;
5967
5968 let result = buffered.await.expect("buffered caller task completes");
5969 assert!(result.is_ok(), "buffered caller observes the flush result");
5970 assert!(
5971 start.elapsed() < Duration::from_secs(1),
5972 "flush must not wait for the one-hour window"
5973 );
5974 let call_count = *client.checkpoint_call_count.lock().unwrap();
5975 assert_eq!(call_count, 1);
5976 }
5977
5978 #[tokio::test(start_paused = true)]
5981 async fn no_delay_checkpoints_write_immediately() {
5982 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5983 let ctx = DurableContext::new_root_with_client(
5984 "arn:test".to_owned(),
5985 lambda_runtime::Context::default(),
5986 Arc::new(CheckpointLog::empty()),
5987 client.clone(),
5988 "token-0".to_owned(),
5989 );
5990
5991 let start = tokio::time::Instant::now();
5992 ctx.checkpoint_updates(vec![make_update("op-1")])
5993 .await
5994 .expect("first write succeeds");
5995 ctx.checkpoint_updates(vec![make_update("op-2")])
5996 .await
5997 .expect("second write succeeds");
5998
5999 assert_eq!(start.elapsed(), Duration::ZERO);
6000 let call_count = *client.checkpoint_call_count.lock().unwrap();
6001 assert_eq!(call_count, 2, "no coalescing without a configured delay");
6002 }
6003
6004 #[derive(Debug)]
6008 struct GatedClient {
6009 inner: InMemoryExecutionClient,
6010 gate: tokio::sync::Semaphore,
6011 calls: std::sync::Mutex<Vec<Vec<String>>>,
6014 }
6015
6016 impl GatedClient {
6017 fn new() -> Self {
6018 Self {
6019 inner: InMemoryExecutionClient::new(Vec::new()),
6020 gate: tokio::sync::Semaphore::new(0),
6021 calls: std::sync::Mutex::new(Vec::new()),
6022 }
6023 }
6024
6025 fn release_one(&self) {
6027 self.gate.add_permits(1);
6028 }
6029
6030 fn call_ids(&self) -> Vec<Vec<String>> {
6031 self.calls
6032 .lock()
6033 .unwrap_or_else(std::sync::PoisonError::into_inner)
6034 .clone()
6035 }
6036 }
6037
6038 impl ExecutionClient for GatedClient {
6039 fn checkpoint(
6040 &self,
6041 execution_arn: &str,
6042 checkpoint_token: &str,
6043 updates: Vec<OperationUpdate>,
6044 ) -> std::pin::Pin<
6045 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6046 > {
6047 let arn = execution_arn.to_owned();
6048 let token = checkpoint_token.to_owned();
6049 Box::pin(async move {
6050 let permit = self.gate.acquire().await.map_err(|e| {
6053 ExecutionClientError::non_retryable(format!("gate closed: {e}"))
6054 })?;
6055 permit.forget();
6056 self.calls
6057 .lock()
6058 .unwrap_or_else(std::sync::PoisonError::into_inner)
6059 .push(updates.iter().map(|u| u.id.clone()).collect());
6060 self.inner.checkpoint(&arn, &token, updates).await
6061 })
6062 }
6063
6064 fn get_state(
6065 &self,
6066 execution_arn: &str,
6067 checkpoint_token: &str,
6068 marker: Option<&str>,
6069 ) -> std::pin::Pin<
6070 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6071 > {
6072 self.inner
6073 .get_state(execution_arn, checkpoint_token, marker)
6074 }
6075 }
6076
6077 #[tokio::test(start_paused = true)]
6083 async fn flush_waits_for_in_flight_timer_claimed_batch() {
6084 let client = Arc::new(GatedClient::new());
6085 let ctx = coalescing_ctx(
6086 Arc::clone(&client) as Arc<dyn ExecutionClient>,
6087 Duration::from_millis(10),
6088 );
6089
6090 let contributor = tokio::spawn({
6093 let ctx = ctx.clone();
6094 async move {
6095 ctx.checkpoint_updates(vec![make_update("op-in-flight")])
6096 .await
6097 }
6098 });
6099 tokio::time::sleep(Duration::from_millis(20)).await;
6102
6103 let mut flush = tokio::spawn({
6108 let ctx = ctx.clone();
6109 async move { ctx.flush_pending_checkpoints().await }
6110 });
6111 let raced = tokio::time::timeout(Duration::from_mins(1), &mut flush).await;
6112 assert!(
6113 raced.is_err(),
6114 "flush returned while the timer-claimed batch write was still in flight — \
6115 the flush barrier is broken"
6116 );
6117
6118 client.release_one();
6120 flush.await.expect("flush task completes");
6121 contributor
6122 .await
6123 .expect("contributor task completes")
6124 .expect("contributor observes the published write");
6125 assert_eq!(
6126 client.call_ids(),
6127 vec![vec!["op-in-flight".to_owned()]],
6128 "exactly the claimed batch was written, once"
6129 );
6130 }
6131
6132 #[derive(Debug)]
6138 struct GatedRetryableFailClient {
6139 gate: tokio::sync::Semaphore,
6140 first: std::sync::atomic::AtomicBool,
6141 calls: std::sync::Mutex<Vec<Vec<String>>>,
6144 }
6145
6146 impl GatedRetryableFailClient {
6147 fn new() -> Self {
6148 Self {
6149 gate: tokio::sync::Semaphore::new(0),
6150 first: std::sync::atomic::AtomicBool::new(true),
6151 calls: std::sync::Mutex::new(Vec::new()),
6152 }
6153 }
6154
6155 fn release_first(&self) {
6157 self.gate.add_permits(1);
6158 }
6159
6160 fn call_ids(&self) -> Vec<Vec<String>> {
6161 self.calls
6162 .lock()
6163 .unwrap_or_else(std::sync::PoisonError::into_inner)
6164 .clone()
6165 }
6166
6167 fn call_count(&self) -> usize {
6168 self.calls
6169 .lock()
6170 .unwrap_or_else(std::sync::PoisonError::into_inner)
6171 .len()
6172 }
6173 }
6174
6175 impl ExecutionClient for GatedRetryableFailClient {
6176 fn checkpoint(
6177 &self,
6178 _execution_arn: &str,
6179 _checkpoint_token: &str,
6180 updates: Vec<OperationUpdate>,
6181 ) -> std::pin::Pin<
6182 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6183 > {
6184 Box::pin(async move {
6185 self.calls
6186 .lock()
6187 .unwrap_or_else(std::sync::PoisonError::into_inner)
6188 .push(updates.iter().map(|u| u.id.clone()).collect());
6189 if self.first.swap(false, std::sync::atomic::Ordering::SeqCst) {
6190 if let Ok(permit) = self.gate.acquire().await {
6194 permit.forget();
6195 }
6196 }
6197 Err(ExecutionClientError::from_retryable(
6198 "injected retryable checkpoint failure".to_owned(),
6199 ))
6200 })
6201 }
6202
6203 fn get_state(
6204 &self,
6205 _execution_arn: &str,
6206 _checkpoint_token: &str,
6207 _marker: Option<&str>,
6208 ) -> std::pin::Pin<
6209 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6210 > {
6211 Box::pin(async {
6212 Err(ExecutionClientError::new_non_retryable(
6213 "get_state not used in this test",
6214 ))
6215 })
6216 }
6217 }
6218
6219 #[tokio::test(start_paused = true)]
6228 async fn queued_batch_flush_after_retryable_failure_writes_nothing() {
6229 let client = Arc::new(GatedRetryableFailClient::new());
6230 let ctx = coalescing_ctx(
6231 Arc::clone(&client) as Arc<dyn ExecutionClient>,
6232 Duration::ZERO,
6233 );
6234
6235 let contributor_a = tokio::spawn({
6238 let ctx = ctx.clone();
6239 async move { ctx.checkpoint_updates(vec![make_update("op-a")]).await }
6240 });
6241 while client.call_count() < 1 {
6242 tokio::time::sleep(Duration::from_millis(1)).await;
6243 }
6244
6245 let contributor_b = tokio::spawn({
6248 let ctx = ctx.clone();
6249 async move { ctx.checkpoint_updates(vec![make_update("op-b")]).await }
6250 });
6251 tokio::time::sleep(Duration::from_millis(5)).await;
6253
6254 client.release_first();
6257
6258 let both = async { tokio::join!(contributor_a, contributor_b) };
6259 assert!(
6260 tokio::time::timeout(Duration::from_secs(60), both)
6261 .await
6262 .is_err(),
6263 "no contributor resolves"
6264 );
6265 assert_eq!(
6266 client.call_ids(),
6267 vec![vec!["op-a".to_owned()]],
6268 "batch B causes no checkpoint call after batch A's failure"
6269 );
6270 assert!(ctx.queued_update_ids().is_empty(), "B's batch is dropped");
6271 let first = ctx.coordinator().write_failure().expect("latched");
6272 assert_eq!(
6273 first.error_type,
6274 crate::error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE
6275 );
6276 assert_eq!(
6277 first.error_message,
6278 "[Unrecoverable Invocation] Checkpoint failed: injected retryable checkpoint failure"
6279 );
6280 }
6281
6282 #[test]
6287 fn checkpoint_failure_fatal_of_a_retryable_error() {
6288 let err = ExecutionClientError::new(ExecutionClientErrorScope::Invocation, "throttled");
6289 let fatal = checkpoint_failure_fatal(&err);
6290 assert_eq!(fatal.error_type, "CheckpointUnrecoverableInvocationError");
6291 assert_eq!(
6292 fatal.error_message,
6293 "[Unrecoverable Invocation] Checkpoint failed: throttled"
6294 );
6295 assert!(matches!(fatal.severity, FatalSeverity::FailInvocation));
6296 }
6297
6298 #[tokio::test(start_paused = true)]
6301 async fn coalesced_fanout_splits_into_size_capped_requests() {
6302 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6303 let limits = BatchLimits {
6304 max_operations: 2,
6305 max_payload_bytes: usize::MAX,
6306 };
6307 let ctx = DurableContext::new_root_with_client_and_coalescer(
6308 "arn:test".to_owned(),
6309 lambda_runtime::Context::default(),
6310 Arc::new(CheckpointLog::empty()),
6311 client.clone(),
6312 "token-0".to_owned(),
6313 CheckpointCoalescer::with_limits(Duration::from_millis(50), limits),
6314 );
6315
6316 let results = tokio::join!(
6318 ctx.checkpoint_updates(vec![make_update("op-1")]),
6319 ctx.checkpoint_updates(vec![make_update("op-2")]),
6320 ctx.checkpoint_updates(vec![make_update("op-3")]),
6321 ctx.checkpoint_updates(vec![make_update("op-4")]),
6322 ctx.checkpoint_updates(vec![make_update("op-5")]),
6323 );
6324 for outcome in [results.0, results.1, results.2, results.3, results.4] {
6325 assert!(outcome.is_ok(), "every contributor observes success");
6326 }
6327
6328 let call_count = *client.checkpoint_call_count.lock().unwrap();
6329 assert_eq!(
6330 call_count, 3,
6331 "five updates under a two-op cap must go out as 2+2+1 requests"
6332 );
6333 let ids: Vec<String> = client
6334 .recorded_updates()
6335 .iter()
6336 .map(|u| u.id.clone())
6337 .collect();
6338 assert_eq!(
6339 ids,
6340 vec!["op-1", "op-2", "op-3", "op-4", "op-5"],
6341 "join order is preserved across the split requests"
6342 );
6343 }
6344
6345 #[tokio::test(start_paused = true)]
6349 async fn batching_mode_batches_writes_behind_in_flight_call() {
6350 let client = Arc::new(GatedClient::new());
6351 let ctx = coalescing_ctx(
6352 Arc::clone(&client) as Arc<dyn ExecutionClient>,
6353 Duration::ZERO,
6354 );
6355
6356 let first = tokio::spawn({
6358 let ctx = ctx.clone();
6359 async move { ctx.checkpoint_updates(vec![make_update("op-first")]).await }
6360 });
6361 tokio::task::yield_now().await;
6362
6363 let second = tokio::spawn({
6366 let ctx = ctx.clone();
6367 async move { ctx.checkpoint_updates(vec![make_update("op-second")]).await }
6368 });
6369 let third = tokio::spawn({
6370 let ctx = ctx.clone();
6371 async move { ctx.checkpoint_updates(vec![make_update("op-third")]).await }
6372 });
6373 tokio::time::sleep(Duration::from_millis(1)).await;
6375
6376 client.release_one();
6378 client.release_one();
6379
6380 for task in [first, second, third] {
6381 task.await
6382 .expect("contributor task completes")
6383 .expect("contributor observes success");
6384 }
6385
6386 assert_eq!(
6387 client.call_ids(),
6388 vec![
6389 vec!["op-first".to_owned()],
6390 vec!["op-second".to_owned(), "op-third".to_owned()],
6391 ],
6392 "the two checkpoints that arrived during the in-flight write \
6393 must batch into one follow-up call"
6394 );
6395 }
6396
6397 #[derive(Clone)]
6408 struct EventCaptureWriter(Arc<std::sync::Mutex<Vec<u8>>>);
6409
6410 impl std::io::Write for EventCaptureWriter {
6411 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
6412 if let Ok(mut inner) = self.0.lock() {
6413 inner.extend_from_slice(buf);
6414 }
6415 Ok(buf.len())
6416 }
6417 fn flush(&mut self) -> std::io::Result<()> {
6418 Ok(())
6419 }
6420 }
6421
6422 impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for EventCaptureWriter {
6423 type Writer = EventCaptureWriter;
6424 fn make_writer(&'a self) -> Self::Writer {
6425 self.clone()
6426 }
6427 }
6428
6429 fn lifecycle_capture() -> (
6432 Arc<std::sync::Mutex<Vec<u8>>>,
6433 tracing::subscriber::DefaultGuard,
6434 ) {
6435 use tracing_subscriber::layer::SubscriberExt as _;
6436 let buffer = Arc::new(std::sync::Mutex::new(Vec::new()));
6437 let fmt_layer = tracing_subscriber::fmt::layer()
6438 .json()
6439 .flatten_event(true)
6440 .with_span_list(true)
6441 .with_writer(EventCaptureWriter(Arc::clone(&buffer)));
6442 let subscriber = tracing_subscriber::registry().with(fmt_layer);
6443 let guard = test_tracing::install_test_subscriber(subscriber);
6444 (buffer, guard)
6445 }
6446
6447 fn captured_lifecycle_events(
6450 buffer: &Arc<std::sync::Mutex<Vec<u8>>>,
6451 event_name: &str,
6452 ) -> Vec<serde_json::Value> {
6453 let output = buffer.lock().map_or_else(
6454 |_| String::new(),
6455 |b| String::from_utf8_lossy(&b).to_string(),
6456 );
6457 output
6458 .lines()
6459 .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
6460 .filter(|v| {
6461 v.get("message").and_then(serde_json::Value::as_str) == Some(event_name)
6462 && v.get("target").and_then(serde_json::Value::as_str)
6463 == Some(crate::observability::TARGET)
6464 })
6465 .collect()
6466 }
6467
6468 #[tokio::test(start_paused = true)]
6482 async fn dropped_buffered_contributor_still_emits_events_for_persisted_updates() {
6483 use crate::observability::event_names;
6484
6485 let mut emitted = false;
6486 for _attempt in 0..5 {
6487 let (buffer, _guard) = lifecycle_capture();
6488 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6489 let ctx = coalescing_ctx(client.clone(), Duration::from_hours(1));
6490
6491 {
6495 let span = tracing::info_span!("contributor-span");
6496 let _entered = span.enter();
6497 let mut fut = Box::pin(ctx.checkpoint_updates(vec![make_update("op-dropped")]));
6498 let mut poll_cx = std::task::Context::from_waker(std::task::Waker::noop());
6499 assert!(
6500 Future::poll(fut.as_mut(), &mut poll_cx).is_pending(),
6501 "the buffered contributor must be awaiting the batch"
6502 );
6503 } ctx.flush_pending_checkpoints().await;
6507
6508 let ids: Vec<String> = client
6509 .recorded_updates()
6510 .iter()
6511 .map(|u| u.id.clone())
6512 .collect();
6513 assert_eq!(
6514 ids,
6515 vec!["op-dropped".to_owned()],
6516 "the dropped contributor's update must still be persisted"
6517 );
6518
6519 let events = captured_lifecycle_events(&buffer, event_names::OPERATION_STARTED);
6520 if events.is_empty() {
6521 continue;
6524 }
6525
6526 let matching: Vec<&serde_json::Value> = events
6527 .iter()
6528 .filter(|v| {
6529 v.get(crate::tracing_layer::fields::OPERATION_ID)
6530 .and_then(serde_json::Value::as_str)
6531 == Some("op-dropped")
6532 })
6533 .collect();
6534 assert_eq!(
6535 matching.len(),
6536 1,
6537 "the persisted transition must emit exactly one operation_started \
6538 event even though its contributor was dropped; got {events:?}"
6539 );
6540 let spans = format!("{:?}", matching.first().and_then(|v| v.get("spans")));
6541 assert!(
6542 spans.contains("contributor-span"),
6543 "the event must carry the originating contributor's span context, \
6544 got spans: {spans}"
6545 );
6546 emitted = true;
6547 break;
6548 }
6549 assert!(
6550 emitted,
6551 "no attempt emitted operation_started for the dropped contributor's \
6552 persisted update — telemetry for a persisted transition was lost"
6553 );
6554 }
6555
6556 #[derive(Debug)]
6560 struct FailSecondCallClient {
6561 inner: InMemoryExecutionClient,
6562 calls: std::sync::Mutex<u32>,
6563 }
6564
6565 impl FailSecondCallClient {
6566 fn new() -> Self {
6567 Self {
6568 inner: InMemoryExecutionClient::new(Vec::new()),
6569 calls: std::sync::Mutex::new(0),
6570 }
6571 }
6572 }
6573
6574 impl ExecutionClient for FailSecondCallClient {
6575 fn checkpoint(
6576 &self,
6577 execution_arn: &str,
6578 checkpoint_token: &str,
6579 updates: Vec<OperationUpdate>,
6580 ) -> std::pin::Pin<
6581 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6582 > {
6583 let arn = execution_arn.to_owned();
6584 let token = checkpoint_token.to_owned();
6585 Box::pin(async move {
6586 let call_index = {
6587 let mut calls = self
6588 .calls
6589 .lock()
6590 .unwrap_or_else(std::sync::PoisonError::into_inner);
6591 *calls += 1;
6592 *calls
6593 };
6594 if call_index >= 2 {
6595 return Err(ExecutionClientError::new_non_retryable(
6596 "injected failure on the second chunk",
6597 ));
6598 }
6599 self.inner.checkpoint(&arn, &token, updates).await
6600 })
6601 }
6602
6603 fn get_state(
6604 &self,
6605 execution_arn: &str,
6606 checkpoint_token: &str,
6607 marker: Option<&str>,
6608 ) -> std::pin::Pin<
6609 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6610 > {
6611 self.inner
6612 .get_state(execution_arn, checkpoint_token, marker)
6613 }
6614 }
6615
6616 #[tokio::test(start_paused = true)]
6627 async fn split_batch_emits_events_for_chunks_persisted_before_a_failure() {
6628 use crate::observability::event_names;
6629
6630 let mut emitted = false;
6631 for _attempt in 0..5 {
6632 let (buffer, _guard) = lifecycle_capture();
6633 let client = Arc::new(FailSecondCallClient::new());
6634 let limits = BatchLimits {
6635 max_operations: 1,
6636 max_payload_bytes: usize::MAX,
6637 };
6638 let ctx = DurableContext::new_root_with_client_and_coalescer(
6639 "arn:test".to_owned(),
6640 lambda_runtime::Context::default(),
6641 Arc::new(CheckpointLog::empty()),
6642 Arc::clone(&client) as Arc<dyn ExecutionClient>,
6643 "token-0".to_owned(),
6644 CheckpointCoalescer::with_limits(Duration::from_millis(50), limits),
6645 );
6646
6647 let first = ctx.submit_checkpoint_updates(vec![make_update("op-persisted")]);
6650 let second = ctx.submit_checkpoint_updates(vec![make_update("op-rejected")]);
6651 until_write_failed(&ctx).await;
6652 let fatal = ctx.coordinator().write_failure().expect("latched");
6653 assert_eq!(
6654 fatal.error_type,
6655 crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
6656 );
6657 let both = async { tokio::join!(first, second) };
6658 assert!(
6659 tokio::time::timeout(Duration::from_secs(5), both)
6660 .await
6661 .is_err(),
6662 "no contributor resolves"
6663 );
6664
6665 let ids: Vec<String> = client
6666 .inner
6667 .recorded_updates()
6668 .iter()
6669 .map(|u| u.id.clone())
6670 .collect();
6671 assert_eq!(
6672 ids,
6673 vec!["op-persisted".to_owned()],
6674 "exactly the first chunk was persisted"
6675 );
6676
6677 let events = captured_lifecycle_events(&buffer, event_names::OPERATION_STARTED);
6678 if events.is_empty() {
6679 continue;
6682 }
6683
6684 let persisted: Vec<&serde_json::Value> = events
6685 .iter()
6686 .filter(|v| {
6687 v.get(crate::tracing_layer::fields::OPERATION_ID)
6688 .and_then(serde_json::Value::as_str)
6689 == Some("op-persisted")
6690 })
6691 .collect();
6692 assert_eq!(
6693 persisted.len(),
6694 1,
6695 "the persisted chunk must emit its operation_started event \
6696 despite the later chunk's failure; got {events:?}"
6697 );
6698 assert!(
6699 !events.iter().any(|v| {
6700 v.get(crate::tracing_layer::fields::OPERATION_ID)
6701 .and_then(serde_json::Value::as_str)
6702 == Some("op-rejected")
6703 }),
6704 "the rejected chunk recorded nothing, so it must emit nothing; \
6705 got {events:?}"
6706 );
6707 emitted = true;
6708 break;
6709 }
6710 assert!(
6711 emitted,
6712 "no attempt emitted operation_started for the persisted chunk — \
6713 telemetry for a persisted transition was lost"
6714 );
6715 }
6716
6717 #[tokio::test]
6729 async fn persisted_checkpoint_emits_events_when_pagination_hydration_fails() {
6730 use crate::observability::event_names;
6731
6732 let mut emitted = false;
6733 for _attempt in 0..5 {
6734 let (buffer, _guard) = lifecycle_capture();
6735 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6736 client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
6739 Vec::new(),
6740 "page-2-token".to_owned(),
6741 ));
6742 client.fail_get_state("injected get_state failure");
6743
6744 let ctx = DurableContext::new_root_with_client(
6745 "arn:test".to_owned(),
6746 lambda_runtime::Context::default(),
6747 Arc::new(CheckpointLog::empty()),
6748 Arc::clone(&client) as Arc<dyn ExecutionClient>,
6749 "token-0".to_owned(),
6750 );
6751
6752 let write = ctx.submit_checkpoint_updates(vec![make_update("op-hydration")]);
6753 until_write_failed(&ctx).await;
6754 let fatal = ctx.coordinator().write_failure().expect("latched");
6755 assert_eq!(
6756 fatal.error_type,
6757 crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
6758 );
6759 assert!(
6760 tokio::time::timeout(Duration::from_millis(100), write)
6761 .await
6762 .is_err(),
6763 "the write's caller never resolves"
6764 );
6765 let ids: Vec<String> = client
6766 .recorded_updates()
6767 .iter()
6768 .map(|u| u.id.clone())
6769 .collect();
6770 assert_eq!(
6771 ids,
6772 vec!["op-hydration".to_owned()],
6773 "the transition was persisted before get_state failed"
6774 );
6775
6776 let events = captured_lifecycle_events(&buffer, event_names::OPERATION_STARTED);
6777 if events.is_empty() {
6778 continue;
6781 }
6782
6783 let matching: Vec<&serde_json::Value> = events
6784 .iter()
6785 .filter(|v| {
6786 v.get(crate::tracing_layer::fields::OPERATION_ID)
6787 .and_then(serde_json::Value::as_str)
6788 == Some("op-hydration")
6789 })
6790 .collect();
6791 assert_eq!(
6792 matching.len(),
6793 1,
6794 "the persisted transition must emit exactly one operation_started \
6795 event even though pagination hydration failed; got {events:?}"
6796 );
6797 emitted = true;
6798 break;
6799 }
6800 assert!(
6801 emitted,
6802 "no attempt emitted operation_started for the persisted transition — \
6803 telemetry was suppressed by a failed pagination fetch"
6804 );
6805 }
6806
6807 async fn until_write_failed(ctx: &DurableContext) {
6809 tokio::time::timeout(Duration::from_secs(5), async {
6810 while !ctx.coordinator().write_failed() {
6811 tokio::time::sleep(Duration::from_millis(1)).await;
6812 }
6813 })
6814 .await
6815 .expect("a write failed");
6816 }
6817
6818 const ADMISSION_BOUND: Duration = Duration::from_secs(1);
6823
6824 fn admission_root(client: &Arc<InMemoryExecutionClient>, window: Duration) -> DurableContext {
6826 DurableContext::new_root_with_client_and_defaults(
6827 "arn:test".to_owned(),
6828 lambda_runtime::Context::default(),
6829 Arc::new(CheckpointLog::empty()),
6830 Arc::clone(client) as Arc<dyn ExecutionClient>,
6831 "token-0".to_owned(),
6832 window,
6833 0,
6834 None,
6835 )
6836 }
6837
6838 fn recorded_ids(client: &InMemoryExecutionClient) -> Vec<String> {
6840 client
6841 .recorded_updates()
6842 .iter()
6843 .map(|u| u.id.clone())
6844 .collect()
6845 }
6846
6847 #[tokio::test]
6850 async fn abandoned_owner_update_is_not_sent() {
6851 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6852 let root = admission_root(&client, Duration::ZERO);
6853 let item = root.new_child("1", None);
6854 root.admission()
6855 .abandon(item.owners().iter().map(String::as_str));
6856 let written = tokio::time::timeout(
6857 ADMISSION_BOUND,
6858 item.checkpoint_updates(vec![make_update("u")]),
6859 )
6860 .await;
6861 assert!(written.is_err(), "a rejected write never resolves");
6862 assert!(client.recorded_updates().is_empty(), "nothing is sent");
6863 }
6864
6865 #[tokio::test]
6866 async fn nested_owner_follows_its_outer_item() {
6867 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6868 let root = admission_root(&client, Duration::ZERO);
6869 let item = root.new_child("1", None);
6870 let inner = item.new_child("1-1", None);
6871 assert_eq!(
6872 inner.owners(),
6873 [
6874 crate::engine::compute_wire_id_public("1"),
6875 crate::engine::compute_wire_id_public("1-1"),
6876 ],
6877 "a nested context runs in its outer item, outermost first"
6878 );
6879 root.admission()
6880 .abandon([crate::engine::compute_wire_id_public("1").as_str()]);
6881 let written = tokio::time::timeout(
6882 ADMISSION_BOUND,
6883 inner.checkpoint_updates(vec![make_update("u")]),
6884 )
6885 .await;
6886 assert!(written.is_err(), "the nested write never resolves");
6887 assert!(client.recorded_updates().is_empty(), "nothing is sent");
6888 }
6889
6890 #[tokio::test(start_paused = true)]
6891 async fn admitted_update_precedes_parent_final_update() {
6892 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6893 let root = admission_root(&client, Duration::from_millis(50));
6894 let item = root.new_child("1", None);
6895 let writer = tokio::spawn({
6896 let item = item.clone();
6897 async move { item.checkpoint_updates(vec![make_update("a")]).await }
6898 });
6899 let coalescer = Arc::clone(&root.inner.coalescer);
6902 let mut yields = 0;
6903 while coalescer.pending_len() == 0 {
6904 assert!(yields < 1_000, "the item update was never buffered");
6905 yields += 1;
6906 tokio::task::yield_now().await;
6907 }
6908 root.admission()
6909 .abandon(item.owners().iter().map(String::as_str));
6910 let parent = tokio::spawn({
6911 let root = root.clone();
6912 async move { root.checkpoint_updates(vec![make_update("p")]).await }
6913 });
6914 tokio::time::advance(Duration::from_millis(100)).await;
6915 tokio::time::timeout(ADMISSION_BOUND, writer)
6916 .await
6917 .expect("the admitted write resolves")
6918 .expect("the writer task completes")
6919 .expect("the admitted write succeeds");
6920 tokio::time::timeout(ADMISSION_BOUND, parent)
6921 .await
6922 .expect("the parent write resolves")
6923 .expect("the parent task completes")
6924 .expect("the parent write succeeds");
6925 assert_eq!(
6926 recorded_ids(&client),
6927 ["a", "p"],
6928 "the admitted item update reaches the service before the parent's"
6929 );
6930 }
6931
6932 #[derive(Debug, Default)]
6936 struct TokenGateClient {
6937 entered: tokio::sync::Notify,
6938 gate: tokio::sync::Notify,
6939 calls: std::sync::Mutex<Vec<(String, Vec<String>)>>,
6940 }
6941
6942 impl TokenGateClient {
6943 fn calls(&self) -> Vec<(String, Vec<String>)> {
6944 self.calls.lock().unwrap().clone()
6945 }
6946 }
6947
6948 impl ExecutionClient for TokenGateClient {
6949 fn checkpoint(
6950 &self,
6951 _arn: &str,
6952 token: &str,
6953 updates: Vec<OperationUpdate>,
6954 ) -> std::pin::Pin<
6955 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6956 > {
6957 let token = token.to_owned();
6958 Box::pin(async move {
6959 let n = {
6960 let mut calls = self.calls.lock().unwrap();
6961 calls.push((token, updates.iter().map(|u| u.id.clone()).collect()));
6962 calls.len()
6963 };
6964 if n == 1 {
6965 self.entered.notify_one();
6966 self.gate.notified().await;
6967 }
6968 Ok(CheckpointOutput {
6969 checkpoint_token: Some(format!("token-{n}")),
6970 updated_operations: Vec::new(),
6971 next_marker: None,
6972 })
6973 })
6974 }
6975
6976 fn get_state(
6977 &self,
6978 _arn: &str,
6979 _token: &str,
6980 _marker: Option<&str>,
6981 ) -> std::pin::Pin<
6982 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6983 > {
6984 Box::pin(async {
6985 Ok(GetStateOutput {
6986 operations: Vec::new(),
6987 next_marker: None,
6988 })
6989 })
6990 }
6991 }
6992
6993 #[tokio::test]
6994 async fn admitted_write_survives_cancelled_caller() {
6995 let client = Arc::new(TokenGateClient::default());
6996 let root = DurableContext::new_root_with_client_and_defaults(
6997 "arn:test".to_owned(),
6998 lambda_runtime::Context::default(),
6999 Arc::new(CheckpointLog::empty()),
7000 Arc::clone(&client) as Arc<dyn ExecutionClient>,
7001 "token-0".to_owned(),
7002 Duration::ZERO,
7003 0,
7004 None,
7005 );
7006 let item = root.new_child("1", None);
7007 let entered = client.entered.notified();
7008 let writer = tokio::spawn({
7009 let item = item.clone();
7010 async move { item.checkpoint_updates(vec![make_update("a")]).await }
7011 });
7012 tokio::time::timeout(ADMISSION_BOUND, entered)
7013 .await
7014 .expect("the client holds the item's write");
7015 writer.abort();
7016 assert!(
7017 writer
7018 .await
7019 .expect_err("the caller was aborted")
7020 .is_cancelled(),
7021 "the caller is cancelled while its write is in flight"
7022 );
7023 client.gate.notify_one();
7024 tokio::time::timeout(
7025 ADMISSION_BOUND,
7026 root.checkpoint_updates(vec![make_update("p")]),
7027 )
7028 .await
7029 .expect("the parent write resolves")
7030 .expect("the parent write succeeds");
7031 assert_eq!(
7032 client.calls(),
7033 [
7034 ("token-0".to_owned(), vec!["a".to_owned()]),
7035 ("token-1".to_owned(), vec!["p".to_owned()]),
7036 ],
7037 "the cancelled caller's write completed and rotated the token"
7038 );
7039 }
7040
7041 #[tokio::test]
7042 async fn abandoned_owner_starts_no_operation() {
7043 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7044 let root = admission_root(&client, Duration::ZERO);
7045 let item = root.new_child("1", None);
7046 root.admission()
7047 .abandon(item.owners().iter().map(String::as_str));
7048 let ran = Arc::new(std::sync::atomic::AtomicBool::new(false));
7049 let flag = Arc::clone(&ran);
7050 let stepped = tokio::time::timeout(
7051 ADMISSION_BOUND,
7052 item.step(move |_| async move {
7053 flag.store(true, std::sync::atomic::Ordering::SeqCst);
7054 Ok(1_u32)
7055 })
7056 .into_future(),
7057 )
7058 .await;
7059 assert!(stepped.is_err(), "the operation never resolves");
7060 assert!(
7061 !ran.load(std::sync::atomic::Ordering::SeqCst),
7062 "the step body never ran"
7063 );
7064 assert!(
7065 client.recorded_updates().is_empty(),
7066 "no update is recorded"
7067 );
7068 }
7069
7070 #[tokio::test]
7073 async fn abandoned_owner_replays_no_operation() {
7074 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7075 let wire = crate::engine::compute_wire_id_public("1-1");
7076 let record = CheckpointRecord {
7077 id: wire.clone(),
7078 status: CheckpointStatus::Succeeded,
7079 result: Some("7".to_owned()),
7080 error_type: None,
7081 error_message: None,
7082 error_data: None,
7083 stack_trace: None,
7084 attempt: 0,
7085 invoke_result: None,
7086 invoke_error_type: None,
7087 invoke_error_message: None,
7088 invoke_error_data: None,
7089 invoke_stack_trace: None,
7090 replay_children: false,
7091 callback_id: None,
7092 op_type: None,
7093 sub_type: None,
7094 due_at: None,
7095 op_name: None,
7096 parent_id: None,
7097 start_timestamp: None,
7098 end_timestamp: None,
7099 };
7100 let log = CheckpointLog::from_records(vec![(wire, record)]);
7101 let root = DurableContext::new_root_with_client_and_defaults(
7102 "arn:test".to_owned(),
7103 lambda_runtime::Context::default(),
7104 Arc::new(log),
7105 Arc::clone(&client) as Arc<dyn ExecutionClient>,
7106 "token-0".to_owned(),
7107 Duration::ZERO,
7108 0,
7109 None,
7110 );
7111 let item = root.new_child("1", None);
7112 root.admission()
7113 .abandon(item.owners().iter().map(String::as_str));
7114 let stepped = tokio::time::timeout(
7115 ADMISSION_BOUND,
7116 item.step(|_| async { Ok(0_u32) }).into_future(),
7117 )
7118 .await;
7119 assert!(
7120 stepped.is_err(),
7121 "the recorded step is not replayed for an abandoned owner: {stepped:?}"
7122 );
7123 }
7124
7125 #[tokio::test]
7132 async fn abandoned_owner_skips_replay_preflight() {
7133 let wait_record = |positional: &str| {
7134 let wire = crate::engine::compute_wire_id_public(positional);
7135 let record = CheckpointRecord {
7136 id: wire.clone(),
7137 status: CheckpointStatus::Succeeded,
7138 result: None,
7139 error_type: None,
7140 error_message: None,
7141 error_data: None,
7142 stack_trace: None,
7143 attempt: 0,
7144 invoke_result: None,
7145 invoke_error_type: None,
7146 invoke_error_message: None,
7147 invoke_error_data: None,
7148 invoke_stack_trace: None,
7149 replay_children: false,
7150 callback_id: None,
7151 op_type: Some("Wait".to_owned()),
7152 sub_type: Some("Wait".to_owned()),
7153 due_at: None,
7154 op_name: None,
7155 parent_id: None,
7156 start_timestamp: None,
7157 end_timestamp: None,
7158 };
7159 (wire, record)
7160 };
7161 let build_root = || {
7162 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7163 let log = CheckpointLog::from_records(vec![wait_record("1-1")]);
7164 let root = DurableContext::new_root_with_client_and_defaults(
7165 "arn:test".to_owned(),
7166 lambda_runtime::Context::default(),
7167 Arc::new(log),
7168 Arc::clone(&client) as Arc<dyn ExecutionClient>,
7169 "token-0".to_owned(),
7170 Duration::ZERO,
7171 0,
7172 None,
7173 );
7174 (client, root)
7175 };
7176
7177 let (_, control_root) = build_root();
7179 let control_item = control_root.new_child("1", None);
7180 let control = tokio::time::timeout(
7181 ADMISSION_BOUND,
7182 control_item.step(|_| async { Ok(0_u32) }).into_future(),
7183 )
7184 .await
7185 .expect("the mismatch resolves at once");
7186 assert!(control.is_err(), "the mismatch is an error: {control:?}");
7187 assert!(
7188 control_item.coordinator().fatal_error().is_some(),
7189 "the mismatch records a fatal error"
7190 );
7191
7192 let (client, root) = build_root();
7194 let item = root.new_child("1", None);
7195 let builder = item.step(|_| async { Ok(0_u32) });
7196 root.admission()
7197 .abandon(item.owners().iter().map(String::as_str));
7198 let stepped = tokio::time::timeout(ADMISSION_BOUND, builder.into_future()).await;
7199 assert!(
7200 stepped.is_err(),
7201 "the abandoned owner's future never resolves: {stepped:?}"
7202 );
7203 assert!(
7204 item.coordinator().fatal_error().is_none(),
7205 "no fatal error is recorded for the abandoned owner"
7206 );
7207 assert!(
7208 root.coordinator().fatal_error().is_none(),
7209 "no fatal error reaches the root"
7210 );
7211 assert!(client.recorded_updates().is_empty(), "nothing is written");
7212 }
7213
7214 #[tokio::test]
7218 async fn abandoned_owner_replays_no_callback() {
7219 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7220 let wire = crate::engine::compute_wire_id_public("1-1");
7221 let record = CheckpointRecord {
7222 id: wire.clone(),
7223 status: CheckpointStatus::Started,
7224 result: None,
7225 error_type: None,
7226 error_message: None,
7227 error_data: None,
7228 stack_trace: None,
7229 attempt: 0,
7230 invoke_result: None,
7231 invoke_error_type: None,
7232 invoke_error_message: None,
7233 invoke_error_data: None,
7234 invoke_stack_trace: None,
7235 replay_children: false,
7236 callback_id: Some("cb-1".to_owned()),
7237 op_type: None,
7238 sub_type: None,
7239 due_at: None,
7240 op_name: None,
7241 parent_id: None,
7242 start_timestamp: None,
7243 end_timestamp: None,
7244 };
7245 let log = Arc::new(CheckpointLog::from_records(vec![(wire, record)]));
7246 let root = DurableContext::new_root_with_client_and_defaults(
7247 "arn:test".to_owned(),
7248 lambda_runtime::Context::default(),
7249 Arc::clone(&log),
7250 Arc::clone(&client) as Arc<dyn ExecutionClient>,
7251 "token-0".to_owned(),
7252 Duration::ZERO,
7253 0,
7254 None,
7255 );
7256 let control = root.new_child("1", None);
7258 let replayed = tokio::time::timeout(
7259 ADMISSION_BOUND,
7260 control.create_callback::<String>().into_future(),
7261 )
7262 .await
7263 .expect("the recorded callback replays")
7264 .expect("the replay succeeds");
7265 assert_eq!(replayed.id(), "cb-1");
7266
7267 let item = root.new_child("1", None);
7268 root.admission()
7269 .abandon(item.owners().iter().map(String::as_str));
7270 let created = tokio::time::timeout(
7271 ADMISSION_BOUND,
7272 item.create_callback::<String>().into_future(),
7273 )
7274 .await;
7275 assert!(
7276 created.is_err(),
7277 "the recorded callback is not replayed for an abandoned owner"
7278 );
7279 assert!(client.recorded_updates().is_empty(), "nothing is sent");
7280 }
7281
7282 #[tokio::test]
7283 async fn admitted_owner_writes_as_before() {
7284 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7285 let root = admission_root(&client, Duration::ZERO);
7286 let item = root.new_child("1", None);
7287 tokio::time::timeout(
7288 ADMISSION_BOUND,
7289 item.checkpoint_updates(vec![make_update("u")]),
7290 )
7291 .await
7292 .expect("the write resolves")
7293 .expect("the write succeeds");
7294 assert_eq!(recorded_ids(&client), ["u"]);
7295 }
7296
7297 #[test]
7300 fn write_without_runtime_returns_invocation_error() {
7301 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7302 let ctx = admission_root(&client, Duration::ZERO);
7303 let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7304 ctx.checkpoint_updates(vec![make_update("u")]).await
7305 });
7306 let std::task::Poll::Ready(Err(err)) = polled else {
7307 unreachable!("expected an immediate error, got {polled:?}");
7308 };
7309 assert_eq!(err.scope(), ExecutionClientErrorScope::Invocation);
7310 assert!(
7311 err.to_string().contains(crate::task::NO_RUNTIME_MESSAGE),
7312 "unexpected error: {err}"
7313 );
7314 assert!(
7315 std::error::Error::source(&err)
7316 .is_some_and(<dyn std::error::Error>::is::<crate::task::NoRuntime>),
7317 "the error carries the no-runtime marker"
7318 );
7319 assert!(client.recorded_updates().is_empty(), "nothing is sent");
7320 }
7321
7322 #[test]
7326 fn operation_without_runtime_returns_spawn_error() {
7327 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7328 let thread_client = Arc::clone(&client);
7329 let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7331 let ctx = admission_root(&thread_client, Duration::ZERO);
7332 let signal = Arc::clone(ctx.coordinator());
7333 (
7334 ctx.step(|_| async { Ok(1_u32) }).into_future().await,
7335 signal,
7336 )
7337 });
7338 let std::task::Poll::Ready((Err(err), signal)) = polled else {
7339 unreachable!("expected an immediate error, got {polled:?}");
7340 };
7341 let spawn_error = crate::task::NoRuntime::operation_error(crate::task::NoRuntime);
7342 assert_eq!(format!("{err:#}"), format!("{spawn_error:#}"));
7343 assert!(client.recorded_updates().is_empty(), "nothing is sent");
7344 assert!(signal.fatal_error().is_none(), "no fault is recorded");
7345 }
7346
7347 #[test]
7352 fn buffered_write_without_runtime_returns_the_no_runtime_error() {
7353 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7354 let thread_client = Arc::clone(&client);
7355 let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7357 let ctx = admission_root(&thread_client, Duration::ZERO);
7358 (ctx.step(|_| async { Ok(1_u32) }).into_future().await, ctx)
7359 });
7360 let std::task::Poll::Ready((Err(err), ctx)) = polled else {
7361 unreachable!("expected an immediate error, got {polled:?}");
7362 };
7363 let spawn_error = crate::task::NoRuntime::operation_error(crate::task::NoRuntime);
7364 assert_eq!(format!("{err:#}"), format!("{spawn_error:#}"));
7365 assert!(client.recorded_updates().is_empty(), "nothing is sent");
7366 assert!(
7367 ctx.coordinator().fatal_error().is_none(),
7368 "no fatal error is recorded"
7369 );
7370 assert!(ctx.queued_update_ids().is_empty(), "the batch is dropped");
7373 let runtime = tokio::runtime::Builder::new_current_thread()
7374 .enable_all()
7375 .build()
7376 .unwrap();
7377 runtime.block_on(ctx.flush_pending_checkpoints());
7378 assert!(
7379 client.recorded_updates().is_empty(),
7380 "the dropped batch is never sent"
7381 );
7382 assert!(ctx.coordinator().fatal_error().is_none());
7383 }
7384
7385 #[test]
7389 fn write_after_failure_without_runtime_never_resolves() {
7390 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7391 let thread_client = Arc::clone(&client);
7392 let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7393 let ctx = admission_root(&thread_client, Duration::ZERO);
7394 ctx.coordinator().fail_write(
7395 FatalError {
7396 error_type: "Failed".to_owned(),
7397 error_message: "failed".to_owned(),
7398 severity: FatalSeverity::FailExecution,
7399 },
7400 || {},
7401 );
7402 ctx.checkpoint_updates(vec![make_update("u")]).await
7403 });
7404 assert!(polled.is_pending(), "{polled:?}");
7405 assert!(client.recorded_updates().is_empty(), "nothing is sent");
7406 }
7407
7408 #[test]
7415 fn no_runtime_error_after_a_failed_write_is_not_published() {
7416 use crate::coordinator::race_points::{self, RacePoint};
7417 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7418 let thread_client = Arc::clone(&client);
7419 let fired = Arc::new(std::sync::atomic::AtomicBool::new(false));
7420 let thread_fired = Arc::clone(&fired);
7421 let polled = crate::task::test_support::poll_once_without_runtime(move || {
7422 let ctx = admission_root(&thread_client, Duration::ZERO);
7423 let failing = ctx.clone();
7424 race_points::set(move |reached| {
7425 if reached == RacePoint::BatchNoRuntimeClaim {
7426 thread_fired.store(true, std::sync::atomic::Ordering::SeqCst);
7427 failing.coordinator().fail_write(
7428 FatalError {
7429 error_type: "Failed".to_owned(),
7430 error_message: "failed".to_owned(),
7431 severity: FatalSeverity::FailExecution,
7432 },
7433 || {},
7434 );
7435 }
7436 });
7437 async move { ctx.checkpoint_updates(vec![make_update("u")]).await }
7438 });
7439 assert!(
7440 fired.load(std::sync::atomic::Ordering::SeqCst),
7441 "the point is reached"
7442 );
7443 assert!(polled.is_pending(), "{polled:?}");
7444 assert!(client.recorded_updates().is_empty(), "nothing is sent");
7445 }
7446
7447 #[test]
7453 fn replayed_child_failure_without_runtime_returns_spawn_error() {
7454 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7455 let wire = crate::engine::compute_wire_id_public("1");
7456 let record = CheckpointRecord {
7457 id: wire.clone(),
7458 status: CheckpointStatus::Started,
7459 result: None,
7460 error_type: None,
7461 error_message: None,
7462 error_data: None,
7463 stack_trace: None,
7464 attempt: 0,
7465 invoke_result: None,
7466 invoke_error_type: None,
7467 invoke_error_message: None,
7468 invoke_error_data: None,
7469 invoke_stack_trace: None,
7470 replay_children: false,
7471 callback_id: None,
7472 op_type: Some("Context".to_owned()),
7473 sub_type: Some(crate::child::CHILD_SUB_TYPE.to_owned()),
7474 due_at: None,
7475 op_name: None,
7476 parent_id: None,
7477 start_timestamp: None,
7478 end_timestamp: None,
7479 };
7480 let thread_client = Arc::clone(&client);
7481 let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7483 let ctx = DurableContext::new_root_with_client_and_defaults(
7484 "arn:test".to_owned(),
7485 lambda_runtime::Context::default(),
7486 Arc::new(CheckpointLog::from_records(vec![(wire, record)])),
7487 thread_client as Arc<dyn ExecutionClient>,
7488 "token-0".to_owned(),
7489 Duration::ZERO,
7490 0,
7491 None,
7492 );
7493 let signal = Arc::clone(ctx.coordinator());
7494 let outcome = ctx
7495 .run_in_child_context(|_child| async { Err::<u32, BoxError>("boom".into()) })
7496 .into_future()
7497 .await;
7498 (outcome, signal)
7499 });
7500 let std::task::Poll::Ready((Err(err), signal)) = polled else {
7501 unreachable!("expected an immediate error, got {polled:?}");
7502 };
7503 let spawn_error = crate::task::NoRuntime::operation_error(crate::task::NoRuntime);
7504 assert_eq!(format!("{err:#}"), format!("{spawn_error:#}"));
7505 assert!(client.recorded_updates().is_empty(), "nothing is sent");
7506 assert!(signal.fatal_error().is_none(), "no fault is recorded");
7507 }
7508
7509 #[derive(Debug)]
7511 struct PanickingClient;
7512
7513 impl ExecutionClient for PanickingClient {
7514 fn checkpoint(
7515 &self,
7516 _arn: &str,
7517 _token: &str,
7518 _updates: Vec<OperationUpdate>,
7519 ) -> std::pin::Pin<
7520 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
7521 > {
7522 #[expect(clippy::panic)] Box::pin(async { panic!("boom in checkpoint") })
7524 }
7525
7526 fn get_state(
7527 &self,
7528 _arn: &str,
7529 _token: &str,
7530 _marker: Option<&str>,
7531 ) -> std::pin::Pin<
7532 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
7533 > {
7534 Box::pin(async {
7535 Ok(GetStateOutput {
7536 operations: Vec::new(),
7537 next_marker: None,
7538 })
7539 })
7540 }
7541 }
7542
7543 async fn write_kept_panic(
7547 client: Arc<dyn ExecutionClient>,
7548 ) -> Box<dyn std::any::Any + Send + 'static> {
7549 let root = DurableContext::new_root_with_client_and_defaults(
7550 "arn:test".to_owned(),
7551 lambda_runtime::Context::default(),
7552 Arc::new(CheckpointLog::empty()),
7553 client,
7554 "token-0".to_owned(),
7555 Duration::ZERO,
7556 0,
7557 None,
7558 );
7559 let coordinator = Arc::clone(root.coordinator());
7560 let mut write = std::pin::pin!(root.checkpoint_updates(vec![make_update("u")]));
7561 let waited = tokio::time::timeout(ADMISSION_BOUND, async {
7562 tokio::select! {
7563 _ = &mut write => false,
7564 () = async {
7565 tokio::task::yield_now().await;
7568 coordinator.tasks().wait_idle().await;
7569 } => true,
7570 }
7571 })
7572 .await
7573 .expect("the write task stops");
7574 assert!(waited, "the caller resolved");
7575 let polled =
7576 std::future::poll_fn(|cx| std::task::Poll::Ready(write.as_mut().poll(cx))).await;
7577 assert!(
7578 polled.is_pending(),
7579 "the caller stays pending after the client panicked"
7580 );
7581 coordinator.take_panic().expect("the panic is kept")
7582 }
7583
7584 #[derive(Debug)]
7586 struct DropPanickingSource;
7587
7588 impl std::fmt::Display for DropPanickingSource {
7589 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
7590 f.write_str("drop-panicking source")
7591 }
7592 }
7593
7594 impl std::error::Error for DropPanickingSource {}
7595
7596 impl Drop for DropPanickingSource {
7597 #[expect(clippy::panic)] fn drop(&mut self) {
7599 panic!("source drop");
7600 }
7601 }
7602
7603 #[derive(Debug)]
7606 struct GatedFailingClient {
7607 release: tokio::sync::Notify,
7608 }
7609
7610 impl ExecutionClient for GatedFailingClient {
7611 fn checkpoint(
7612 &self,
7613 _arn: &str,
7614 _token: &str,
7615 _updates: Vec<OperationUpdate>,
7616 ) -> std::pin::Pin<
7617 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
7618 > {
7619 Box::pin(async {
7620 self.release.notified().await;
7621 Err(ExecutionClientError::with_source(
7622 ExecutionClientErrorScope::Invocation,
7623 "gated failure",
7624 DropPanickingSource,
7625 ))
7626 })
7627 }
7628
7629 fn get_state(
7630 &self,
7631 _arn: &str,
7632 _token: &str,
7633 _marker: Option<&str>,
7634 ) -> std::pin::Pin<
7635 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
7636 > {
7637 Box::pin(async {
7638 Ok(GetStateOutput {
7639 operations: Vec::new(),
7640 next_marker: None,
7641 })
7642 })
7643 }
7644 }
7645
7646 #[tokio::test]
7651 async fn unread_write_result_drop_panic_is_recorded() {
7652 let client = Arc::new(GatedFailingClient {
7653 release: tokio::sync::Notify::new(),
7654 });
7655 let root = DurableContext::new_root_with_client_and_defaults(
7656 "arn:test".to_owned(),
7657 lambda_runtime::Context::default(),
7658 Arc::new(CheckpointLog::empty()),
7659 Arc::clone(&client) as Arc<dyn ExecutionClient>,
7660 "token-0".to_owned(),
7661 Duration::ZERO,
7662 0,
7663 None,
7664 );
7665 let coordinator = Arc::clone(root.coordinator());
7666 let mut write = Box::pin(root.checkpoint_updates(vec![make_update("u")]));
7667 let polled =
7668 std::future::poll_fn(|cx| std::task::Poll::Ready(write.as_mut().poll(cx))).await;
7669 assert!(polled.is_pending());
7670 drop(write);
7671 client.release.notify_one();
7672 tokio::time::timeout(ADMISSION_BOUND, coordinator.tasks().wait_idle())
7673 .await
7674 .expect("the write task stops");
7675 let payload = coordinator
7676 .take_panic()
7677 .expect("the destructor's panic is recorded");
7678 assert_eq!(
7679 crate::task::test_support::str_payload(payload.as_ref()),
7680 Some("source drop")
7681 );
7682 }
7683
7684 #[tokio::test]
7688 async fn write_panic_is_recorded_and_caller_stays_pending() {
7689 let payload = write_kept_panic(Arc::new(PanickingClient)).await;
7690 assert_eq!(
7691 crate::task::test_support::str_payload(payload.as_ref()),
7692 Some("boom in checkpoint")
7693 );
7694 }
7695
7696 struct ReadyThenDropPanic;
7699
7700 impl Future for ReadyThenDropPanic {
7701 type Output = Result<CheckpointOutput, ExecutionClientError>;
7702
7703 fn poll(
7704 self: std::pin::Pin<&mut Self>,
7705 _cx: &mut std::task::Context<'_>,
7706 ) -> std::task::Poll<Self::Output> {
7707 std::task::Poll::Ready(Ok(CheckpointOutput {
7708 checkpoint_token: Some("token-1".to_owned()),
7709 updated_operations: Vec::new(),
7710 next_marker: None,
7711 }))
7712 }
7713 }
7714
7715 impl Drop for ReadyThenDropPanic {
7716 #[expect(clippy::panic)] fn drop(&mut self) {
7718 panic!("boom in checkpoint drop");
7719 }
7720 }
7721
7722 #[derive(Debug)]
7724 struct DropPanickingClient;
7725
7726 impl ExecutionClient for DropPanickingClient {
7727 fn checkpoint(
7728 &self,
7729 _arn: &str,
7730 _token: &str,
7731 _updates: Vec<OperationUpdate>,
7732 ) -> std::pin::Pin<
7733 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
7734 > {
7735 Box::pin(ReadyThenDropPanic)
7736 }
7737
7738 fn get_state(
7739 &self,
7740 _arn: &str,
7741 _token: &str,
7742 _marker: Option<&str>,
7743 ) -> std::pin::Pin<
7744 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
7745 > {
7746 Box::pin(async {
7747 Ok(GetStateOutput {
7748 operations: Vec::new(),
7749 next_marker: None,
7750 })
7751 })
7752 }
7753 }
7754
7755 #[tokio::test]
7759 async fn write_drop_panic_is_recorded_and_caller_stays_pending() {
7760 let payload = write_kept_panic(Arc::new(DropPanickingClient)).await;
7761 assert_eq!(
7762 crate::task::test_support::str_payload(payload.as_ref()),
7763 Some("boom in checkpoint drop")
7764 );
7765 }
7766
7767 #[tokio::test]
7768 async fn child_context_owner_covers_its_descendants() {
7769 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7770 let root = admission_root(&client, Duration::ZERO);
7771 let child = root.new_child("1", None);
7772 root.admission()
7773 .abandon([crate::engine::compute_wire_id_public("1").as_str()]);
7774 let written = tokio::time::timeout(
7775 ADMISSION_BOUND,
7776 child.checkpoint_updates(vec![make_update("u")]),
7777 )
7778 .await;
7779 assert!(written.is_err(), "the child's write never resolves");
7780 tokio::time::timeout(
7781 ADMISSION_BOUND,
7782 root.checkpoint_updates(vec![make_update("p")]),
7783 )
7784 .await
7785 .expect("the root write resolves")
7786 .expect("the root write succeeds");
7787 assert_eq!(recorded_ids(&client), ["p"], "only the root update is sent");
7788 }
7789
7790 #[tokio::test]
7791 async fn finished_child_rejects_escaped_descendant_write() {
7792 use aws_sdk_lambda::types::OperationAction;
7793
7794 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7795 let root = admission_root(&client, Duration::ZERO);
7796 let slot: Arc<std::sync::Mutex<Option<DurableFuture<u32>>>> =
7797 Arc::new(std::sync::Mutex::new(None));
7798 let release = Arc::new(tokio::sync::Notify::new());
7799 let step_wire = crate::engine::compute_wire_id_public("1-1");
7800 let child_wire = crate::engine::compute_wire_id_public("1");
7801
7802 let returned = tokio::time::timeout(ADMISSION_BOUND, {
7803 let slot = Arc::clone(&slot);
7804 let release = Arc::clone(&release);
7805 let client = Arc::clone(&client);
7806 let step_wire = step_wire.clone();
7807 root.run_in_child_context(move |child| async move {
7808 let handle = child
7809 .step(move |_| async move {
7810 release.notified().await;
7811 Ok(7_u32)
7812 })
7813 .name("s")
7814 .spawn();
7815 *slot.lock().unwrap() = Some(handle);
7816 let mut yields = 0;
7817 while !client
7818 .recorded_updates()
7819 .iter()
7820 .any(|u| u.id == step_wire && u.action == OperationAction::Start)
7821 {
7822 if yields == 1_000 {
7823 return Err("the step START was never recorded".into());
7824 }
7825 yields += 1;
7826 tokio::task::yield_now().await;
7827 }
7828 Ok(1_u32)
7829 })
7830 .into_future()
7831 })
7832 .await
7833 .expect("the child returns")
7834 .expect("the child succeeds");
7835 assert_eq!(returned, 1);
7836
7837 release.notify_one();
7838 let handle = slot
7839 .lock()
7840 .unwrap()
7841 .take()
7842 .expect("the body stored the handle");
7843 let awaited = tokio::time::timeout(ADMISSION_BOUND, handle).await;
7844 assert!(awaited.is_err(), "the escaped operation never resolves");
7845
7846 let updates = client.recorded_updates();
7847 let has = |id: &str, action: OperationAction| {
7848 updates.iter().any(|u| u.id == id && u.action == action)
7849 };
7850 assert!(
7851 has(&step_wire, OperationAction::Start),
7852 "s START is recorded"
7853 );
7854 assert!(
7855 has(&child_wire, OperationAction::Succeed),
7856 "the child's SUCCEED is recorded"
7857 );
7858 assert!(
7859 !has(&step_wire, OperationAction::Succeed),
7860 "s SUCCEED is never sent after the child returned"
7861 );
7862 }
7863
7864 fn tenant_span_lists(
7870 tenant_id: Option<&str>,
7871 ) -> (Vec<serde_json::Value>, Vec<serde_json::Value>) {
7872 use crate::tracing_layer::tests::{json_records, json_span_subscriber, spans_of};
7873
7874 let buffer = Arc::new(std::sync::Mutex::new(Vec::new()));
7875 let subscriber = json_span_subscriber(&buffer);
7876 {
7877 let _guard = test_tracing::install_test_subscriber(subscriber);
7878 let mut lambda_context = lambda_runtime::Context::default();
7879 lambda_context.tenant_id = tenant_id.map(str::to_owned);
7880 let root = DurableContext::new_root(
7881 "arn:test".to_owned(),
7882 lambda_context,
7883 Arc::new(CheckpointLog::empty()),
7884 );
7885 root.replay_span().in_scope(|| tracing::info!("in-root"));
7886 let child = root.new_child("1", Some("child"));
7887 child.replay_span().in_scope(|| tracing::info!("in-child"));
7888 };
7889 let records = json_records(&buffer);
7890 (
7891 spans_of(&records, "in-root"),
7892 spans_of(&records, "in-child"),
7893 )
7894 }
7895
7896 #[test]
7897 fn tenant_id_on_root_and_child_spans() {
7898 use crate::tracing_layer::tests::has_field;
7899
7900 let tenant = serde_json::Value::String("t-1".to_owned());
7901 let (root, child) = tenant_span_lists(Some("t-1"));
7902 assert!(
7903 has_field(&root, crate::tracing_layer::fields::TENANT_ID, &tenant),
7904 "{root:?}"
7905 );
7906 assert!(
7907 has_field(&child, crate::tracing_layer::fields::TENANT_ID, &tenant),
7908 "{child:?}"
7909 );
7910 }
7911
7912 #[test]
7913 fn no_tenant_id_without_tenant() {
7914 use crate::tracing_layer::tests::any_field;
7915
7916 let (root, child) = tenant_span_lists(None);
7917 assert!(
7918 !any_field(&root, crate::tracing_layer::fields::TENANT_ID),
7919 "{root:?}"
7920 );
7921 assert!(
7922 !any_field(&child, crate::tracing_layer::fields::TENANT_ID),
7923 "{child:?}"
7924 );
7925 assert!(
7927 any_field(&child, crate::tracing_layer::fields::OPERATION_ID),
7928 "{child:?}"
7929 );
7930 }
7931}
7932
7933#[cfg(test)]
7934#[expect(clippy::disallowed_methods)]
7937mod coordinator_tests {
7938 use super::{DurableContext, remaining_until};
7939 use crate::client::ExecutionClient;
7940 use crate::coordinator::{InvocationCoordinator, PermitKind, Phase};
7941 use crate::driver::test_support::{ScriptedClient, operation_like};
7942 use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
7943 use crate::task::test_support::{Schedule, ScheduleState};
7944 use aws_sdk_lambda::types::{OperationAction, OperationStatus, OperationType, OperationUpdate};
7945 use std::cell::RefCell;
7946 use std::sync::Arc;
7947 use std::sync::atomic::{AtomicBool, Ordering};
7948 use std::time::{Duration, SystemTime};
7949
7950 fn ctx_with(log: Arc<CheckpointLog>, client: &Arc<ScriptedClient>) -> DurableContext {
7951 DurableContext::new_root_with_client(
7952 "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
7953 lambda_runtime::Context::default(),
7954 log,
7955 Arc::clone(client) as Arc<dyn ExecutionClient>,
7956 "t0".to_owned(),
7957 )
7958 }
7959
7960 #[test]
7961 fn remaining_time_cases() {
7962 let now = SystemTime::now();
7963 let now_ms = |offset: i64| {
7964 let base = now
7965 .duration_since(std::time::UNIX_EPOCH)
7966 .expect("after the epoch")
7967 .as_millis();
7968 u64::try_from(i128::try_from(base).expect("fits") + i128::from(offset))
7969 .expect("non-negative")
7970 };
7971 assert_eq!(remaining_until(0, now), None);
7972 let ahead = remaining_until(now_ms(5_000), now).expect("a deadline");
7973 assert!(ahead > Duration::from_secs(4) && ahead <= Duration::from_secs(5));
7974 assert_eq!(remaining_until(now_ms(-1_000), now), Some(Duration::ZERO));
7975
7976 let ctx = DurableContext::test_context();
7978 assert_eq!(ctx.remaining_time(), None, "the default deadline is 0");
7979 }
7980
7981 #[tokio::test(start_paused = true)]
7985 async fn wait_for_record_without_record_starts_as_started() {
7986 let ctx = DurableContext::test_context();
7987 let coordinator = Arc::clone(ctx.coordinator());
7988 let registered = tokio::time::Instant::now();
7989 let wire = "missing".to_owned();
7990 let mut wait = Box::pin(ctx.wait_for_record(&wire, |s| *s == CheckpointStatus::Succeeded));
7991 let polled = futures_poll_once(wait.as_mut());
7992 assert!(polled.is_pending());
7993 assert_eq!(
7994 coordinator.next_refresh_at(),
7995 Some(registered + Duration::from_secs(1))
7996 );
7997 tokio::time::advance(Duration::from_secs(1)).await;
7998 let crate::coordinator::DueRefresh::Due { keys, .. } =
7999 coordinator.take_due(tokio::time::Instant::now())
8000 else {
8001 unreachable!("the registration is due");
8002 };
8003 let records =
8004 std::collections::BTreeMap::from([(wire.clone(), (CheckpointStatus::Started, None))]);
8005 coordinator.after_refresh(&keys, &records);
8006 assert_eq!(
8007 coordinator.next_refresh_at(),
8008 Some(registered + Duration::from_secs(2)),
8009 "an unchanged STARTED record backs off"
8010 );
8011 }
8012
8013 fn futures_poll_once<F: Future + ?Sized>(
8015 fut: std::pin::Pin<&mut F>,
8016 ) -> std::task::Poll<F::Output> {
8017 let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
8018 fut.poll(&mut cx)
8019 }
8020
8021 #[test]
8022 fn wait_for_record_ready_before_registration() {
8023 let wire = crate::engine::compute_wire_id_public("1");
8024 let log = Arc::new(CheckpointLog::from_records(vec![(
8025 wire.clone(),
8026 CheckpointRecord::with_status(&wire, CheckpointStatus::Succeeded),
8027 )]));
8028 let client = Arc::new(ScriptedClient::empty());
8029 let ctx = ctx_with(log, &client);
8030 let coordinator = Arc::clone(ctx.coordinator());
8031 let schedule = Schedule::idle(&coordinator);
8032 let mut wait = Box::pin(ctx.wait_for_record(&wire, |s| *s != CheckpointStatus::Started));
8033 assert!(schedule.poll(wait.as_mut()).is_ready());
8034 assert_eq!(coordinator.live_waits(), 0);
8035 assert_eq!(client.call_count(), 0);
8036 }
8037
8038 type Hook = Option<(Arc<CheckpointLog>, Arc<InvocationCoordinator>, String)>;
8041
8042 thread_local! {
8043 static HOOK: RefCell<Hook> = const { RefCell::new(None) };
8044 }
8045
8046 fn merge_during_first_read(status: &CheckpointStatus) -> bool {
8051 if let Some((log, coordinator, wire)) = HOOK.with(|hook| hook.borrow_mut().take()) {
8052 log.insert(
8053 wire.clone(),
8054 CheckpointRecord::with_status(&wire, CheckpointStatus::Succeeded),
8055 );
8056 coordinator.wake_operations(&[wire]);
8057 return false;
8058 }
8059 *status != CheckpointStatus::Started
8060 }
8061
8062 #[test]
8063 fn wait_for_record_merge_after_read_is_not_lost() {
8064 let wire = crate::engine::compute_wire_id_public("1");
8065 let log = Arc::new(CheckpointLog::from_records(vec![(
8066 wire.clone(),
8067 CheckpointRecord::with_status(&wire, CheckpointStatus::Started),
8068 )]));
8069 let client = Arc::new(ScriptedClient::empty());
8070 let ctx = ctx_with(Arc::clone(&log), &client);
8071 let coordinator = Arc::clone(ctx.coordinator());
8072 HOOK.with(|hook| {
8073 *hook.borrow_mut() = Some((Arc::clone(&log), Arc::clone(&coordinator), wire.clone()));
8074 });
8075 let schedule = Schedule::idle(&coordinator);
8076 let mut wait = Box::pin(ctx.wait_for_record(&wire, merge_during_first_read));
8077 assert!(schedule.poll(wait.as_mut()).is_pending());
8078 assert_eq!(
8079 schedule.state(),
8080 ScheduleState::Queued,
8081 "the wake during the poll keeps the task queued"
8082 );
8083 assert_eq!(coordinator.permits(PermitKind::Activity), 1);
8084 assert_eq!(coordinator.phase(), Phase::Running);
8085 assert!(schedule.poll(wait.as_mut()).is_ready());
8086 assert_eq!(
8087 client.call_count(),
8088 0,
8089 "no checkpoint call between the polls"
8090 );
8091 assert_eq!(coordinator.live_waits(), 0);
8092 }
8093
8094 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8095 async fn write_holds_token_lock_through_response_merge() {
8096 let gate = Arc::new(tokio::sync::Notify::new());
8097 let ctx_slot: Arc<std::sync::OnceLock<DurableContext>> =
8098 Arc::new(std::sync::OnceLock::new());
8099 let a_merged_at_b = Arc::new(AtomicBool::new(false));
8100 let responder_ctx = Arc::clone(&ctx_slot);
8101 let responder_flag = Arc::clone(&a_merged_at_b);
8102 let client = Arc::new(
8103 ScriptedClient::new(move |updates, history| {
8104 if history.len() == updates.len() {
8105 return updates
8107 .iter()
8108 .map(|u| {
8109 operation_like(u, OperationStatus::Succeeded)
8110 .build()
8111 .expect("all required fields set")
8112 })
8113 .collect();
8114 }
8115 let merged = responder_ctx
8116 .get()
8117 .is_some_and(|ctx| ctx.has_checkpoint_record("1"));
8118 responder_flag.store(merged, Ordering::SeqCst);
8119 Vec::new()
8120 })
8121 .with_first_call_gate(Arc::clone(&gate)),
8122 );
8123 let ctx = ctx_with(Arc::new(CheckpointLog::empty()), &client);
8124 ctx_slot.set(ctx.clone()).expect("set once");
8125 let update = |positional: &str| {
8126 OperationUpdate::builder()
8127 .id(crate::engine::compute_wire_id_public(positional))
8128 .r#type(OperationType::Step)
8129 .sub_type("Step")
8130 .action(OperationAction::Start)
8131 .build()
8132 .expect("all required fields set")
8133 };
8134 let a_ctx = ctx.clone();
8135 let a_update = update("1");
8136 let a = tokio::spawn(async move { a_ctx.checkpoint_updates_direct(vec![a_update]).await });
8137 for _ in 0..1000 {
8138 if client.call_count() == 1 {
8139 break;
8140 }
8141 tokio::time::sleep(Duration::from_millis(1)).await;
8142 }
8143 assert_eq!(client.call_count(), 1, "A's call was received");
8144 let b_ctx = ctx.clone();
8145 let b_update = update("2");
8146 let b = tokio::spawn(async move { b_ctx.checkpoint_updates_direct(vec![b_update]).await });
8147 for _ in 0..100 {
8148 tokio::task::yield_now().await;
8149 }
8150 assert_eq!(client.call_count(), 1, "B waits for A's merge");
8151 gate.notify_one();
8152 a.await.expect("A does not panic").expect("A succeeds");
8153 b.await.expect("B does not panic").expect("B succeeds");
8154 let calls = client.calls();
8155 assert_eq!(calls.len(), 2);
8156 assert_eq!(
8157 calls.get(1).map(|(token, _)| token.as_str()),
8158 Some("t1"),
8159 "B's call carries the token A's response rotated in"
8160 );
8161 assert!(
8162 a_merged_at_b.load(Ordering::SeqCst),
8163 "operation a was merged before B's call was sent"
8164 );
8165 }
8166
8167 #[tokio::test]
8168 async fn hold_activity_after_suspension_never_resolves() {
8169 let ctx = DurableContext::test_context();
8170 ctx.coordinator().finish();
8171 let ran = Arc::new(AtomicBool::new(false));
8172 let held = Arc::clone(&ran);
8173 let waited = tokio::time::timeout(
8174 Duration::from_millis(100),
8175 ctx.hold_activity(async move { held.store(true, Ordering::SeqCst) }),
8176 )
8177 .await;
8178 assert!(waited.is_err(), "the held future never resolves");
8179 assert!(!ran.load(Ordering::SeqCst), "the work never ran");
8180 }
8181
8182 #[tokio::test]
8183 async fn write_after_suspension_is_rejected() {
8184 let client = Arc::new(ScriptedClient::empty());
8185 let ctx = ctx_with(Arc::new(CheckpointLog::empty()), &client);
8186 ctx.coordinator().finish();
8187 let update = OperationUpdate::builder()
8188 .id("u")
8189 .r#type(OperationType::Step)
8190 .action(OperationAction::Start)
8191 .build()
8192 .expect("all required fields set");
8193 let written = tokio::time::timeout(
8194 Duration::from_millis(100),
8195 ctx.checkpoint_updates_direct(vec![update]),
8196 )
8197 .await;
8198 assert!(written.is_err(), "the rejected write never resolves");
8199 assert_eq!(client.call_count(), 0, "nothing was sent");
8200 }
8201
8202 #[tokio::test]
8203 async fn wait_for_record_after_suspension_never_resolves() {
8204 let wire = crate::engine::compute_wire_id_public("1");
8205 let log = Arc::new(CheckpointLog::from_records(vec![(
8206 wire.clone(),
8207 CheckpointRecord::with_status(&wire, CheckpointStatus::Succeeded),
8208 )]));
8209 let ctx = ctx_with(log, &Arc::new(ScriptedClient::empty()));
8210 ctx.coordinator().finish();
8211 let waited = tokio::time::timeout(
8212 Duration::from_millis(100),
8213 ctx.wait_for_record(&wire, |s| *s != CheckpointStatus::Started),
8214 )
8215 .await;
8216 assert!(waited.is_err(), "a refused registration never resolves");
8217 }
8218
8219 fn retry_update(positional: &str, sub_type: &str, payload: &str) -> OperationUpdate {
8221 OperationUpdate::builder()
8222 .id(crate::engine::compute_wire_id_public(positional))
8223 .r#type(OperationType::Step)
8224 .sub_type(sub_type)
8225 .action(OperationAction::Retry)
8226 .payload(payload)
8227 .build()
8228 .expect("all required fields set")
8229 }
8230
8231 #[tokio::test]
8232 async fn wfc_retry_keeps_status_the_response_carries() {
8233 let wfc = crate::wait_for_condition::WFC_SUB_TYPE;
8234 for (carried, expected) in [
8235 (OperationStatus::Ready, CheckpointStatus::Ready),
8236 (OperationStatus::Succeeded, CheckpointStatus::Succeeded),
8237 (OperationStatus::Failed, CheckpointStatus::Failed),
8238 ] {
8239 let wire = crate::engine::compute_wire_id_public("1");
8240 let log = Arc::new(CheckpointLog::from_records(vec![(
8241 wire.clone(),
8242 CheckpointRecord::with_status(&wire, CheckpointStatus::Started),
8243 )]));
8244 let status = carried.clone();
8245 let client = Arc::new(ScriptedClient::new(move |updates, _| {
8246 updates
8247 .iter()
8248 .map(|u| {
8249 operation_like(u, status.clone())
8250 .build()
8251 .expect("all required fields set")
8252 })
8253 .collect()
8254 }));
8255 let ctx = ctx_with(Arc::clone(&log), &client);
8256 ctx.checkpoint_updates_direct(vec![retry_update("1", wfc, "1")])
8257 .await
8258 .expect("the write succeeds");
8259 assert_eq!(
8260 log.get(&wire).expect("present").status,
8261 expected,
8262 "a RETRY response that carries {carried:?} keeps it"
8263 );
8264 }
8265 }
8266
8267 #[tokio::test]
8268 async fn wfc_retry_omitted_from_response_is_pending() {
8269 let wfc = crate::wait_for_condition::WFC_SUB_TYPE;
8270 let ready = crate::engine::compute_wire_id_public("1");
8271 let pending = crate::engine::compute_wire_id_public("2");
8272 let step = crate::engine::compute_wire_id_public("3");
8273 let succeeded = crate::engine::compute_wire_id_public("5");
8274 let mut pending_record = CheckpointRecord::with_status(&pending, CheckpointStatus::Pending);
8275 pending_record.result = Some("old".to_owned());
8276 let log = Arc::new(CheckpointLog::from_records(vec![
8277 (
8278 ready.clone(),
8279 CheckpointRecord::with_status(&ready, CheckpointStatus::Ready),
8280 ),
8281 (pending.clone(), pending_record),
8282 (
8283 step.clone(),
8284 CheckpointRecord::with_status(&step, CheckpointStatus::Started),
8285 ),
8286 (
8287 succeeded.clone(),
8288 CheckpointRecord::with_status(&succeeded, CheckpointStatus::Started),
8289 ),
8290 ]));
8291 let client = Arc::new(ScriptedClient::empty());
8292 let ctx = ctx_with(Arc::clone(&log), &client);
8293 ctx.checkpoint_updates_direct(vec![
8294 retry_update("1", wfc, "next"),
8295 retry_update("2", wfc, "new"),
8296 retry_update("3", "Step", "x"),
8297 retry_update("4", wfc, "x"),
8298 OperationUpdate::builder()
8299 .id(crate::engine::compute_wire_id_public("5"))
8300 .r#type(OperationType::Step)
8301 .sub_type(wfc)
8302 .action(OperationAction::Succeed)
8303 .payload("y")
8304 .build()
8305 .expect("all required fields set"),
8306 ])
8307 .await
8308 .expect("the write succeeds");
8309
8310 let record = log.get(&ready).expect("present");
8311 assert_eq!(
8312 record.status,
8313 CheckpointStatus::Pending,
8314 "a READY record from the previous cycle takes the PENDING the service recorded"
8315 );
8316 assert_eq!(record.result.as_deref(), Some("next"));
8317 assert_eq!(
8318 log.get(&pending).expect("present").result.as_deref(),
8319 Some("old"),
8320 "a record the service already marked PENDING is kept"
8321 );
8322 assert_eq!(
8323 log.get(&step).expect("present").status,
8324 CheckpointStatus::Started,
8325 "a step RETRY is not a wait_for_condition RETRY"
8326 );
8327 assert_eq!(
8328 log.get(&succeeded).expect("present").status,
8329 CheckpointStatus::Started,
8330 "a wait_for_condition SUCCEED is not a RETRY"
8331 );
8332 let missing = crate::engine::compute_wire_id_public("4");
8333 assert!(
8334 log.get(&missing).is_none(),
8335 "a missing record stays missing"
8336 );
8337 }
8338}
8339
8340#[cfg(test)]
8342#[expect(clippy::disallowed_methods)]
8344mod write_order_tests {
8345 use std::sync::Arc;
8346 use std::time::Duration;
8347
8348 use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
8349
8350 use super::DurableContext;
8351 use crate::client::{
8352 CheckpointOutput, ExecutionClient, ExecutionClientError, GetStateOutput,
8353 InMemoryExecutionClient,
8354 };
8355 use crate::engine::CheckpointLog;
8356
8357 fn start(id: &str) -> OperationUpdate {
8358 OperationUpdate::builder()
8359 .id(id.to_owned())
8360 .r#type(OperationType::Step)
8361 .action(OperationAction::Start)
8362 .build()
8363 .expect("all required OperationUpdate fields set")
8364 }
8365
8366 fn context(client: &Arc<InMemoryExecutionClient>, window: Duration) -> DurableContext {
8368 DurableContext::new_root_with_client_and_defaults(
8369 "arn:test".to_owned(),
8370 lambda_runtime::Context::default(),
8371 Arc::new(CheckpointLog::empty()),
8372 Arc::clone(client) as Arc<dyn ExecutionClient>,
8373 "token-0".to_owned(),
8374 window,
8375 0,
8376 None,
8377 )
8378 }
8379
8380 fn ids(client: &InMemoryExecutionClient) -> Vec<String> {
8381 client
8382 .recorded_updates()
8383 .iter()
8384 .map(|u| u.id().to_owned())
8385 .collect()
8386 }
8387
8388 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
8389 async fn submissions_reach_client_in_submission_order() {
8390 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8391 let ctx = context(&client, Duration::ZERO);
8392 tokio::spawn(async move {
8393 let a = ctx.submit_checkpoint_updates(vec![start("A")]);
8394 let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8395 a.await.expect("A is written");
8396 b.await.expect("B is written");
8397 })
8398 .await
8399 .expect("the submitting task completes");
8400 assert_eq!(ids(&client), ["A", "B"]);
8401 }
8402
8403 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8404 async fn child_and_root_submissions_share_write_order() {
8405 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8406 let root = context(&client, Duration::ZERO);
8407 let hold_a = Arc::new(tokio::sync::Notify::new());
8408 let gate_notify = Arc::clone(&hold_a);
8409 root.set_pre_lock_gate(Arc::new(move |updates: &[OperationUpdate]| {
8410 updates
8411 .iter()
8412 .any(|u| u.id() == "A")
8413 .then(|| Arc::clone(&gate_notify))
8414 }));
8415 let child = root.new_child("1", None);
8416 let a = child.submit_checkpoint_updates(vec![start("A")]);
8417 let b = root.submit_checkpoint_updates(vec![start("B")]);
8418 let deadline = tokio::time::Instant::now() + Duration::from_millis(500);
8419 while tokio::time::Instant::now() < deadline {
8420 assert!(
8421 client.recorded_updates().is_empty(),
8422 "a write reached the client while A was held: {:?}",
8423 ids(&client)
8424 );
8425 tokio::time::sleep(Duration::from_millis(10)).await;
8426 }
8427 hold_a.notify_one();
8428 a.await.expect("A is written");
8429 b.await.expect("B is written");
8430 assert_eq!(ids(&client), ["A", "B"]);
8431 }
8432
8433 #[tokio::test]
8434 async fn dropping_reserved_write_releases_in_order() {
8435 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8436 let ctx = context(&client, Duration::ZERO);
8437 let a = ctx.reserve_checkpoint_updates(vec![start("A")]);
8438 let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8439 drop(a);
8440 tokio::time::timeout(Duration::from_secs(2), b)
8441 .await
8442 .map_err(|_| "B is blocked behind the dropped A")
8443 .unwrap()
8444 .expect("B is written");
8445 assert_eq!(ids(&client), ["A", "B"]);
8446 }
8447
8448 #[tokio::test]
8451 async fn released_reservation_is_written_before_later_submissions() {
8452 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8453 let ctx = context(&client, Duration::ZERO);
8454 let a = ctx.reserve_checkpoint_updates(vec![start("A")]);
8455 let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8456 tokio::time::sleep(Duration::from_millis(50)).await;
8457 assert!(
8458 client.recorded_updates().is_empty(),
8459 "nothing is sent while A is held"
8460 );
8461 let a = a.release();
8462 a.await.expect("A is written");
8463 b.await.expect("B is written");
8464 assert_eq!(ids(&client), ["A", "B"]);
8465 }
8466
8467 #[tokio::test]
8472 async fn fatal_record_keeps_admitted_writes() {
8473 for window in [Duration::ZERO, Duration::from_secs(60)] {
8474 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8475 let ctx = context(&client, window);
8476 let a = ctx.submit_checkpoint_updates(vec![start("A")]);
8477 ctx.coordinator()
8478 .record_fatal("Error".to_owned(), "boom".to_owned());
8479 let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8480 ctx.flush_pending_checkpoints().await;
8481 tokio::time::timeout(Duration::from_secs(2), a)
8482 .await
8483 .map_err(|_| format!("A does not resolve ({window:?})"))
8484 .unwrap()
8485 .expect("A is written");
8486 assert_eq!(ids(&client), ["A"], "{window:?}");
8487 assert!(
8488 tokio::time::timeout(Duration::from_millis(50), b)
8489 .await
8490 .is_err(),
8491 "a refused write never resolves ({window:?})"
8492 );
8493 assert_eq!(ids(&client), ["A"], "{window:?}");
8494 }
8495 }
8496
8497 #[tokio::test]
8500 async fn rejected_reservation_sends_nothing() {
8501 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8503 let ctx = context(&client, Duration::ZERO);
8504 ctx.coordinator().finish();
8505 let write = ctx.reserve_checkpoint_updates(vec![start("A")]).release();
8506 assert!(
8507 tokio::time::timeout(Duration::from_millis(50), write)
8508 .await
8509 .is_err(),
8510 "a refused reservation never resolves"
8511 );
8512 assert!(client.recorded_updates().is_empty(), "nothing is sent");
8513 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8515 let root = context(&client, Duration::ZERO);
8516 let child = root.new_child("1", None);
8517 root.admission()
8518 .abandon([crate::engine::compute_wire_id_public("1").as_str()]);
8519 let write = child.reserve_checkpoint_updates(vec![start("A")]).release();
8520 assert!(
8521 tokio::time::timeout(Duration::from_millis(50), write)
8522 .await
8523 .is_err(),
8524 "a rejected join never resolves"
8525 );
8526 assert_eq!(client.recorded_updates(), Vec::new());
8527 }
8528
8529 #[derive(Default)]
8532 struct PanicsOnA(std::sync::Mutex<Vec<String>>);
8533
8534 impl ExecutionClient for PanicsOnA {
8535 fn checkpoint(
8536 &self,
8537 _execution_arn: &str,
8538 _checkpoint_token: &str,
8539 updates: Vec<OperationUpdate>,
8540 ) -> std::pin::Pin<
8541 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
8542 > {
8543 assert!(!updates.iter().any(|u| u.id() == "A"), "client boom");
8545 self.0
8546 .lock()
8547 .unwrap()
8548 .extend(updates.iter().map(|u| u.id().to_owned()));
8549 Box::pin(async {
8550 Ok(CheckpointOutput::new(
8551 Some("t".to_owned()),
8552 Vec::new(),
8553 None,
8554 ))
8555 })
8556 }
8557
8558 fn get_state(
8559 &self,
8560 _execution_arn: &str,
8561 _checkpoint_token: &str,
8562 _marker: Option<&str>,
8563 ) -> std::pin::Pin<
8564 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
8565 > {
8566 Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
8567 }
8568 }
8569
8570 #[tokio::test]
8573 async fn write_after_flusher_panic_is_sent() {
8574 let client = Arc::new(PanicsOnA::default());
8575 let ctx = DurableContext::new_root_with_client(
8576 "arn:test".to_owned(),
8577 lambda_runtime::Context::default(),
8578 Arc::new(CheckpointLog::empty()),
8579 Arc::clone(&client) as Arc<dyn ExecutionClient>,
8580 "token-0".to_owned(),
8581 );
8582 let a = ctx.submit_checkpoint_updates(vec![start("A")]);
8583 assert!(
8584 tokio::time::timeout(Duration::from_millis(50), a)
8585 .await
8586 .is_err(),
8587 "the panicked write never resolves"
8588 );
8589 assert!(
8590 ctx.coordinator().has_panic(),
8591 "the gateway recorded the panic"
8592 );
8593 let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8594 tokio::time::timeout(Duration::from_secs(2), b)
8595 .await
8596 .expect("a new writer sends B")
8597 .expect("B is written");
8598 assert_eq!(*client.0.lock().unwrap(), ["B"]);
8599 }
8600}
8601
8602pub(crate) fn foreign_caller_message(method: &str) -> String {
8605 format!(
8606 "context usage error in \"{method}\": the operation was called on a task or thread \
8607 that does not own the context. Start concurrent operations with .spawn(), map, \
8608 parallel or a combinator, not with tokio::spawn or std::thread::spawn."
8609 )
8610}
8611
8612#[cfg(test)]
8615mod change_hook_tests {
8616 use super::{CheckpointCoalescer, DurableContext};
8617 use crate::OperationStatus;
8618 use crate::client::{ExecutionClient, InMemoryExecutionClient, TestResponse};
8619 use crate::driver::test_support::{
8620 RefreshClient, ScriptedClient, drive_handler, named, operation_like,
8621 };
8622 use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
8623 use crate::plugin::test_support::Changes;
8624 use crate::plugin::{InvocationPlugins, OperationInfo, PluginFactory};
8625 use aws_sdk_lambda::types::{
8626 Operation, OperationAction, OperationStatus as WireStatus, OperationType, OperationUpdate,
8627 };
8628 use std::sync::{Arc, Mutex, OnceLock};
8629 use std::time::Duration;
8630
8631 fn ctx_on(
8633 client: Arc<dyn ExecutionClient>,
8634 log: Arc<CheckpointLog>,
8635 window: Duration,
8636 plugins: Arc<InvocationPlugins>,
8637 ) -> DurableContext {
8638 DurableContext::new_root_with_client_and_defaults(
8639 "arn:test".to_owned(),
8640 lambda_runtime::Context::default(),
8641 log,
8642 client,
8643 "t0".to_owned(),
8644 window,
8645 0,
8646 Some(plugins),
8647 )
8648 }
8649
8650 fn echo_client() -> Arc<ScriptedClient> {
8653 Arc::new(ScriptedClient::new(|updates, _history| {
8654 updates
8655 .iter()
8656 .map(|u| {
8657 let status = match u.action() {
8658 OperationAction::Succeed => WireStatus::Succeeded,
8659 OperationAction::Fail => WireStatus::Failed,
8660 OperationAction::Retry => WireStatus::Pending,
8661 _ => WireStatus::Started,
8662 };
8663 operation_like(u, status).build().expect("all fields set")
8664 })
8665 .collect()
8666 }))
8667 }
8668
8669 fn step_op(id: &str, status: WireStatus) -> Operation {
8671 Operation::builder()
8672 .id(id)
8673 .r#type(OperationType::Step)
8674 .status(status)
8675 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
8676 .build()
8677 .expect("all fields set")
8678 }
8679
8680 fn start(id: &str) -> OperationUpdate {
8682 OperationUpdate::builder()
8683 .id(id)
8684 .r#type(OperationType::Step)
8685 .action(OperationAction::Start)
8686 .build()
8687 .expect("all fields set")
8688 }
8689
8690 #[tokio::test]
8693 async fn change_hook_runs_outside_locks() {
8694 type Locks = (Arc<tokio::sync::Mutex<String>>, Arc<CheckpointCoalescer>);
8695 let locks: Arc<OnceLock<Locks>> = Arc::default();
8696 let free: Arc<Mutex<Vec<(bool, bool)>>> = Arc::default();
8697 let mut recorder = Changes::default();
8698 {
8699 let locks = Arc::clone(&locks);
8700 let free = Arc::clone(&free);
8701 recorder.on_change = Some(Arc::new(move |_info| {
8702 let (token, coalescer) = locks.get().expect("set before the run");
8703 let token_free = token.try_lock().is_ok();
8704 let writer_free = coalescer.writer_lock().try_lock().is_ok();
8705 free.lock().unwrap().push((token_free, writer_free));
8706 }));
8707 }
8708 let ctx = ctx_on(
8709 echo_client(),
8710 Arc::new(CheckpointLog::empty()),
8711 Duration::ZERO,
8712 recorder.plugins(),
8713 );
8714 locks
8715 .set((
8716 ctx.checkpoint_token_mutex(),
8717 Arc::clone(&ctx.inner.coalescer),
8718 ))
8719 .map_err(|_| "set twice")
8720 .expect("set once");
8721 drive_handler(ctx, None, |ctx| async move {
8722 ctx.step(|_| async { Ok(1_u32) })
8723 .name("s")
8724 .await
8725 .map(|v| v.to_string())
8726 .map_err(|e| crate::error::WireError::new(None::<String>, Some(e.to_string())))
8727 })
8728 .await
8729 .expect("no panic");
8730 let free = free.lock().unwrap().clone();
8731 assert!(!free.is_empty(), "at least one call");
8732 assert!(
8733 free.iter().all(|&(token, writer)| token && writer),
8734 "{free:?}"
8735 );
8736 }
8737
8738 #[tokio::test]
8742 async fn retained_start_write_reports_its_change() {
8743 let recorder = Changes::default();
8744 let seen: Arc<Mutex<Vec<(OperationAction, usize)>>> = Arc::default();
8745 let client = {
8746 let calls = Arc::clone(&recorder.calls);
8747 let seen = Arc::clone(&seen);
8748 Arc::new(ScriptedClient::new(move |updates, _history| {
8749 let delivered = calls.lock().unwrap().len();
8750 updates
8751 .iter()
8752 .map(|u| {
8753 seen.lock().unwrap().push((u.action().clone(), delivered));
8754 let status = if u.action() == &OperationAction::Succeed {
8755 WireStatus::Succeeded
8756 } else {
8757 WireStatus::Started
8758 };
8759 operation_like(u, status).build().expect("all fields set")
8760 })
8761 .collect()
8762 }))
8763 };
8764 let ctx = ctx_on(
8765 client,
8766 Arc::new(CheckpointLog::empty()),
8767 Duration::ZERO,
8768 recorder.plugins(),
8769 );
8770 drive_handler(ctx, None, |ctx| async move {
8771 ctx.step(|_| async { Ok(1_u32) })
8772 .name("s")
8773 .await
8774 .map(|v| v.to_string())
8775 .map_err(|e| crate::error::WireError::new(None::<String>, Some(e.to_string())))
8776 })
8777 .await
8778 .expect("no panic");
8779 let seen = seen.lock().unwrap().clone();
8780 assert_eq!(
8781 seen,
8782 [(OperationAction::Start, 0), (OperationAction::Succeed, 1)],
8783 "{:?}",
8784 recorder.timeline()
8785 );
8786 }
8787
8788 struct HeldWrite {
8792 ctx: DurableContext,
8793 recorder: Changes,
8794 write: std::pin::Pin<Box<super::SubmittedWrite>>,
8795 release: std::sync::mpsc::Sender<()>,
8796 holder: Option<std::thread::JoinHandle<()>>,
8797 }
8798
8799 impl HeldWrite {
8800 fn start() -> Self {
8801 Self::start_with(Ok(crate::client::CheckpointOutput::new(
8802 None,
8803 Vec::new(),
8804 None,
8805 )))
8806 }
8807
8808 fn start_with(
8810 result: Result<crate::client::CheckpointOutput, crate::client::ExecutionClientError>,
8811 ) -> Self {
8812 let (entered_tx, entered_rx) = std::sync::mpsc::channel::<()>();
8813 let (release, release_rx) = std::sync::mpsc::channel::<()>();
8814 let release_rx = Mutex::new(release_rx);
8815 let first = std::sync::atomic::AtomicBool::new(true);
8816 let recorder = Changes {
8817 on_change: Some(Arc::new(move |_info| {
8818 if first.swap(false, std::sync::atomic::Ordering::SeqCst) {
8819 entered_tx.send(()).unwrap();
8820 release_rx.lock().unwrap().recv().unwrap();
8821 }
8822 })),
8823 ..Changes::default()
8824 };
8825 let ctx = ctx_on(
8826 Arc::new(InMemoryExecutionClient::new(Vec::new())),
8827 Arc::new(CheckpointLog::empty()),
8828 Duration::ZERO,
8829 recorder.plugins(),
8830 );
8831 let entry = |id: &str| {
8832 vec![OperationInfo::from_record(
8833 id,
8834 &CheckpointRecord::with_status(id, CheckpointStatus::Started),
8835 )]
8836 };
8837 let plugins = ctx.shared_plugins().expect("the context has plugins");
8838 plugins.push_operation_change(entry("earlier"));
8839 let holder = {
8840 let ctx = ctx.clone();
8841 std::thread::spawn(move || {
8842 ctx.plugins()
8843 .expect("the context has plugins")
8844 .drain_operation_changes(&ctx.inner.engine.checkpoint_log);
8845 })
8846 };
8847 entered_rx.recv().unwrap();
8848 plugins.push_operation_change(entry("own"));
8850 let (sender, receiver) = tokio::sync::oneshot::channel();
8851 let write = Box::pin(super::SubmittedWrite::from_receiver(receiver).reported_by(&ctx));
8852 assert!(sender.send(result).is_ok(), "the write is waiting");
8853 Self {
8854 ctx,
8855 recorder,
8856 write,
8857 release,
8858 holder: Some(holder),
8859 }
8860 }
8861
8862 fn join_holder(&mut self) {
8864 self.holder
8865 .take()
8866 .expect("the holder is joined once")
8867 .join()
8868 .unwrap();
8869 }
8870
8871 fn waiter_count(&self) -> usize {
8872 self.ctx
8873 .plugins()
8874 .expect("the context has plugins")
8875 .delivery_waiter_count()
8876 }
8877 }
8878
8879 #[test]
8887 fn write_waits_for_change_delivered_by_another_task() {
8888 let mut held = HeldWrite::start();
8889 let (signal, signals) = crate::plugin::test_support::SignalWaker::new();
8890 let waker = std::task::Waker::from(Arc::clone(&signal));
8891 let mut cx = std::task::Context::from_waker(&waker);
8892 for _ in 0..100 {
8893 assert!(
8894 held.write.as_mut().poll(&mut cx).is_pending(),
8895 "the write waits while its change set is undelivered"
8896 );
8897 }
8898 assert_eq!(held.waiter_count(), 1);
8899 assert_eq!(held.recorder.updated_ids(), [["earlier"]]);
8900 assert_eq!(signal.woken(), 0);
8901 held.release.send(()).unwrap();
8902 signals
8903 .recv_timeout(Duration::from_secs(30))
8904 .expect("a delivery woke the write's waker");
8905 held.join_holder();
8906 match held.write.as_mut().poll(&mut cx) {
8907 std::task::Poll::Ready(result) => result.expect("the write succeeded"),
8908 std::task::Poll::Pending => unreachable!("the write is ready once delivered"),
8909 };
8910 assert_eq!(held.recorder.updated_ids(), [["earlier"], ["own"]]);
8911 assert_eq!(held.waiter_count(), 0);
8912 }
8913
8914 #[test]
8917 fn dropped_write_removes_its_registration() {
8918 let mut held = HeldWrite::start();
8919 let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
8920 assert!(held.write.as_mut().poll(&mut cx).is_pending());
8921 assert_eq!(held.waiter_count(), 1);
8922 held.write = Box::pin(super::SubmittedWrite::rejected());
8923 assert_eq!(held.waiter_count(), 0);
8924 held.release.send(()).unwrap();
8925 held.join_holder();
8926 assert_eq!(held.recorder.updated_ids(), [["earlier"], ["own"]]);
8927 assert_eq!(held.waiter_count(), 0);
8928 }
8929
8930 async fn change_work_of_run(plugins: Option<Arc<InvocationPlugins>>) -> (usize, usize) {
8933 let ctx = DurableContext::new_root_with_client_and_defaults(
8934 "arn:test".to_owned(),
8935 lambda_runtime::Context::default(),
8936 Arc::new(CheckpointLog::empty()),
8937 echo_client(),
8938 "t0".to_owned(),
8939 Duration::ZERO,
8940 0,
8941 plugins,
8942 );
8943 let projections = crate::engine::projections();
8944 let infos = crate::plugin::infos_from_records();
8945 drive_handler(ctx, None, |ctx| async move {
8946 let one = ctx.step(|_| async { Ok(1_u32) }).name("s").await;
8947 let two = ctx
8948 .run_in_child_context(|child| async move {
8949 Ok(child.step(|_| async { Ok(2_u32) }).name("inner").await?)
8950 })
8951 .name("c")
8952 .await;
8953 Ok(format!("{one:?} {two:?}"))
8954 })
8955 .await
8956 .expect("no panic");
8957 (
8958 crate::engine::projections() - projections,
8959 crate::plugin::infos_from_records() - infos,
8960 )
8961 }
8962
8963 #[tokio::test]
8967 async fn no_plugin_builds_no_change_info() {
8968 assert_eq!(change_work_of_run(None).await, (0, 0));
8969 let (projected, infos) = change_work_of_run(Some(Changes::default().plugins())).await;
8970 assert!(projected > 0, "a plugin's run projects the log");
8971 assert!(infos > 0, "a plugin's run builds infos");
8972 }
8973
8974 #[tokio::test]
8979 async fn write_reports_change_on_caller() {
8980 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8981 client.enqueue_checkpoint_response(TestResponse::Success(vec![step_op(
8982 "d",
8983 WireStatus::Started,
8984 )]));
8985 client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
8986 vec![step_op("d", WireStatus::Succeeded)],
8987 "page-2-token".to_owned(),
8988 ));
8989 client.fail_get_state("injected get_state failure");
8990 let recorder = Changes::default();
8991 let ctx = ctx_on(
8992 client,
8993 Arc::new(CheckpointLog::empty()),
8994 Duration::ZERO,
8995 recorder.plugins(),
8996 );
8997 ctx.checkpoint_updates_direct(vec![start("d")])
8998 .await
8999 .expect("the write succeeds");
9000 assert_eq!(recorder.updated_ids(), [["d"]]);
9001 let failed = ctx.checkpoint_updates_direct(vec![start("d")]);
9002 assert!(
9003 tokio::time::timeout(Duration::from_millis(100), failed)
9004 .await
9005 .is_err(),
9006 "the failed write never resolves"
9007 );
9008 assert!(ctx.coordinator().write_failed());
9009 ctx.report_operation_changes().await;
9010 assert_eq!(recorder.updated_ids(), [["d"], ["d"]]);
9011 }
9012
9013 #[tokio::test]
9016 async fn unchanged_response_queues_nothing() {
9017 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9018 client.enqueue_checkpoint_response(TestResponse::Success(vec![step_op(
9019 "s",
9020 WireStatus::Started,
9021 )]));
9022 let recorder = Changes::default();
9023 let log = Arc::new(CheckpointLog::from_records(vec![(
9024 "s".to_owned(),
9025 CheckpointRecord::with_status("s", CheckpointStatus::Started),
9026 )]));
9027 let ctx = ctx_on(client, log, Duration::ZERO, recorder.plugins());
9028 ctx.checkpoint_updates(vec![start("s")])
9029 .await
9030 .expect("the write succeeds");
9031 assert!(recorder.calls().is_empty(), "{:?}", recorder.updated_ids());
9032 }
9033
9034 #[tokio::test]
9037 async fn execution_listed_once_per_change_set() {
9038 let execution = |status: WireStatus| {
9039 Operation::builder()
9040 .id("exec-1")
9041 .r#type(OperationType::Execution)
9042 .status(status)
9043 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
9044 .build()
9045 .expect("all fields set")
9046 };
9047 let client = Arc::new(InMemoryExecutionClient::new(vec![execution(
9048 WireStatus::Succeeded,
9049 )]));
9050 client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
9051 vec![execution(WireStatus::Started)],
9052 "page-2-token".to_owned(),
9053 ));
9054 let recorder = Changes::default();
9055 let ctx = ctx_on(
9056 client,
9057 Arc::new(CheckpointLog::empty()),
9058 Duration::ZERO,
9059 recorder.plugins(),
9060 );
9061 ctx.checkpoint_updates(Vec::new())
9062 .await
9063 .expect("the write succeeds");
9064 let calls = recorder.calls();
9065 assert_eq!(calls.len(), 1, "{:?}", recorder.updated_ids());
9066 let info = calls.first().expect("one call");
9067 let statuses: Vec<_> = info
9068 .updated_operations()
9069 .iter()
9070 .map(OperationInfo::status)
9071 .collect();
9072 assert_eq!(statuses, [Some(OperationStatus::Succeeded)]);
9073 }
9074
9075 #[tokio::test]
9078 async fn change_reported_when_pagination_hydration_fails() {
9079 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9080 client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
9081 vec![step_op("op-hydration", WireStatus::Started)],
9082 "page-2-token".to_owned(),
9083 ));
9084 client.fail_get_state("injected get_state failure");
9085 let recorder = Changes::default();
9086 let ctx = ctx_on(
9087 client,
9088 Arc::new(CheckpointLog::empty()),
9089 Duration::ZERO,
9090 recorder.plugins(),
9091 );
9092 let write = ctx.checkpoint_updates(vec![start("op-hydration")]);
9093 assert!(
9094 tokio::time::timeout(Duration::from_millis(100), write)
9095 .await
9096 .is_err(),
9097 "the failed write never resolves"
9098 );
9099 assert!(ctx.coordinator().write_failed());
9100 ctx.report_operation_changes().await;
9101 assert_eq!(recorder.updated_ids(), [["op-hydration"]]);
9102 }
9103
9104 #[tokio::test(start_paused = true)]
9109 async fn refresh_reports_wait_change() {
9110 let client = Arc::new(RefreshClient::new(|req| {
9111 if req.empty_number == 1
9112 && let Some(start) = named(req.history, "w", &OperationAction::Start)
9113 {
9114 let record = operation_like(start, WireStatus::Succeeded)
9115 .build()
9116 .expect("all required fields set");
9117 return Ok(vec![record]);
9118 }
9119 Ok(Vec::new())
9120 }));
9121 let recorder = Changes::default();
9122 let ctx = ctx_on(
9123 Arc::clone(&client) as Arc<dyn ExecutionClient>,
9124 Arc::new(CheckpointLog::empty()),
9125 Duration::ZERO,
9126 recorder.plugins(),
9127 );
9128 let outcome = drive_handler(ctx, None, |ctx| async move {
9129 let wait = ctx.wait(Duration::from_secs(1)).name("w").spawn();
9130 let step = ctx
9131 .step(|_| async {
9132 tokio::time::sleep(Duration::from_secs(3)).await;
9133 Ok(1_u32)
9134 })
9135 .name("s")
9136 .spawn();
9137 let (wait, _step) = tokio::join!(wait, step);
9138 Ok(format!("{}", wait.is_ok()))
9139 })
9140 .await
9141 .expect("no panic");
9142 assert_eq!(
9143 outcome,
9144 crate::driver::InvocationOutcome::Complete("true".to_owned())
9145 );
9146 let succeeded = recorder.calls().iter().any(|info| {
9147 info.updated_operations()
9148 .iter()
9149 .any(|op| op.name() == Some("w") && op.status() == Some(OperationStatus::Succeeded))
9150 });
9151 assert!(succeeded, "{:?}", recorder.timeline());
9152 let timeline = recorder.timeline();
9153 let change = timeline
9154 .iter()
9155 .position(|e| e == "change:w:SUCCEEDED")
9156 .expect("the change is recorded");
9157 let end = timeline
9158 .iter()
9159 .position(|e| e == "end:w")
9160 .expect("the end hook ran");
9161 assert!(change < end, "{timeline:?}");
9162 }
9163
9164 #[tokio::test(start_paused = true)]
9170 async fn refresh_sender_reports_change() {
9171 let client = Arc::new(RefreshClient::new(|req| {
9172 let Some(start) = named(req.history, "w", &OperationAction::Start) else {
9173 return Ok(Vec::new());
9174 };
9175 let record = match req.empty_number {
9176 1 => step_op("x", WireStatus::Succeeded),
9177 2 => operation_like(start, WireStatus::Succeeded)
9178 .build()
9179 .expect("all required fields set"),
9180 _ => return Ok(Vec::new()),
9181 };
9182 Ok(vec![record])
9183 }));
9184 let at: Arc<Mutex<Vec<Duration>>> = Arc::default();
9185 let mut recorder = Changes::default();
9186 {
9187 let at = Arc::clone(&at);
9188 let started = tokio::time::Instant::now();
9189 recorder.on_change = Some(Arc::new(move |info| {
9190 if info.updated_operations().iter().any(|op| op.id() == "x") {
9191 at.lock().unwrap().push(started.elapsed());
9192 }
9193 }));
9194 }
9195 let ctx = ctx_on(
9196 Arc::clone(&client) as Arc<dyn ExecutionClient>,
9197 Arc::new(CheckpointLog::empty()),
9198 Duration::ZERO,
9199 recorder.plugins(),
9200 );
9201 drive_handler(ctx, None, |ctx| async move {
9202 let wait = ctx.wait(Duration::from_secs(1)).name("w").spawn();
9203 let step = ctx
9204 .step(|_| async {
9205 tokio::time::sleep(Duration::from_secs(3)).await;
9206 Ok(1_u32)
9207 })
9208 .name("s")
9209 .spawn();
9210 let (wait, _step) = tokio::join!(wait, step);
9211 Ok(format!("{}", wait.is_ok()))
9212 })
9213 .await
9214 .expect("no panic");
9215 let at = at.lock().unwrap().clone();
9216 assert_eq!(at.len(), 1, "{:?}", recorder.timeline());
9217 assert!(
9218 at.first()
9219 .is_some_and(|t| *t < Duration::from_millis(1_500)),
9220 "the change of x is reported at the first refresh: {at:?}, {:?}",
9221 client.empty_answers()
9222 );
9223 }
9224
9225 async fn projections_of_run(plugin_count: usize) -> (usize, usize) {
9229 let records = (0..2_000)
9230 .map(|i| {
9231 let id = format!("old-{i}");
9232 let mut record = CheckpointRecord::with_status(&id, CheckpointStatus::Succeeded);
9233 record.op_type = Some("STEP".to_owned());
9234 (id, record)
9235 })
9236 .collect();
9237 let recorders: Vec<Changes> = (0..plugin_count).map(|_| Changes::default()).collect();
9238 let plugins = crate::plugin::test_support::plugins_of_all(
9239 recorders
9240 .iter()
9241 .map(|r| Arc::new(r.clone()) as Arc<dyn PluginFactory>)
9242 .collect(),
9243 );
9244 let ctx = ctx_on(
9245 echo_client(),
9246 Arc::new(CheckpointLog::from_records(records)),
9247 Duration::ZERO,
9248 plugins,
9249 );
9250 let before = crate::engine::projections();
9251 drive_handler(ctx, None, |ctx| async move {
9252 for i in 0..50_u32 {
9253 ctx.step(move |_| async move { Ok(i) })
9254 .name(format!("new-{i}"))
9255 .await
9256 .map_err(|e| {
9257 crate::error::WireError::new(None::<String>, Some(e.to_string()))
9258 })?;
9259 }
9260 Ok("done".to_owned())
9261 })
9262 .await
9263 .expect("no panic");
9264 let projected = crate::engine::projections() - before;
9265 let calls = recorders.first().expect("a plugin").calls().len();
9266 (projected, calls)
9267 }
9268
9269 #[tokio::test]
9272 async fn change_hooks_project_once_per_response() {
9273 let (one_projected, one_calls) = projections_of_run(1).await;
9274 let (three_projected, three_calls) = projections_of_run(3).await;
9275 assert!(one_calls > 0);
9276 assert_eq!(one_projected, one_calls);
9277 assert_eq!(three_projected, three_calls);
9278 assert_eq!(one_calls, three_calls);
9279 }
9280
9281 #[tokio::test]
9284 async fn execution_update_replaces_slot() {
9285 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9286 let execution = Operation::builder()
9287 .id("exec-1")
9288 .r#type(OperationType::Execution)
9289 .status(WireStatus::Succeeded)
9290 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
9291 .build()
9292 .expect("all fields set");
9293 client.enqueue_checkpoint_response(TestResponse::Success(vec![
9294 execution,
9295 step_op("s", WireStatus::Started),
9296 ]));
9297 let recorder = Changes::default();
9298 let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
9299 vec![("test::Changes", Arc::new(recorder.clone()))];
9300 let slot = OperationInfo::execution(
9301 "exec-1",
9302 "EXECUTION",
9303 &CheckpointStatus::Started,
9304 None,
9305 None,
9306 None,
9307 );
9308 let plugins =
9309 crate::plugin::create_plugins(&factories, slot, crate::plugin::tests_support_info)
9310 .expect("one factory");
9311 let log = Arc::new(CheckpointLog::empty());
9312 let ctx = ctx_on(client, Arc::clone(&log), Duration::ZERO, plugins);
9313 ctx.checkpoint_updates(vec![start("s")])
9314 .await
9315 .expect("the write succeeds");
9316 let calls = recorder.calls();
9317 let info = calls.first().expect("one call");
9318 assert_eq!(
9319 info.operations()
9320 .get("exec-1")
9321 .and_then(OperationInfo::status),
9322 Some(OperationStatus::Succeeded)
9323 );
9324 assert!(
9325 info.updated_operations()
9326 .iter()
9327 .any(|op| op.id() == "exec-1")
9328 );
9329 assert!(!log.contains("exec-1"));
9330 }
9331
9332 #[tokio::test]
9335 async fn paginated_response_is_one_change_set() {
9336 let client = Arc::new(InMemoryExecutionClient::new(vec![
9337 step_op("step-1", WireStatus::Succeeded),
9338 step_op("step-2", WireStatus::Succeeded),
9339 ]));
9340 client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
9341 vec![step_op("step-1", WireStatus::Started)],
9342 "page-2-token".to_owned(),
9343 ));
9344 let recorder = Changes::default();
9345 let ctx = ctx_on(
9346 client,
9347 Arc::new(CheckpointLog::empty()),
9348 Duration::ZERO,
9349 recorder.plugins(),
9350 );
9351 ctx.checkpoint_updates(Vec::new())
9352 .await
9353 .expect("the write succeeds");
9354 assert_eq!(recorder.updated_ids(), [["step-1", "step-2"]]);
9355 let calls = recorder.calls();
9356 let step_1 = calls
9357 .first()
9358 .and_then(|info| info.updated_operations().first())
9359 .expect("step-1 is listed");
9360 assert_eq!(step_1.status(), Some(OperationStatus::Succeeded));
9361 }
9362
9363 #[tokio::test]
9372 async fn merged_status_is_hidden_until_its_change_is_queued() {
9373 for case in ["unpaginated", "paginated", "failed page read"] {
9374 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9375 let response = vec![step_op("d", WireStatus::Succeeded)];
9376 client.enqueue_checkpoint_response(if case == "unpaginated" {
9377 TestResponse::Success(response)
9378 } else {
9379 TestResponse::SuccessPaginated(response, "page-2-token".to_owned())
9380 });
9381 if case == "failed page read" {
9382 client.fail_get_state("injected get_state failure");
9383 }
9384 let recorder = Changes::default();
9385 let log = Arc::new(CheckpointLog::from_records(vec![(
9386 "d".to_owned(),
9387 CheckpointRecord::with_status("d", CheckpointStatus::Started),
9388 )]));
9389 let ctx = ctx_on(client, log, Duration::ZERO, recorder.plugins());
9390 let (poll_tx, poll_rx) = std::sync::mpsc::channel::<()>();
9394 let (reply_tx, reply_rx) = std::sync::mpsc::channel::<()>();
9395 let resolved: Arc<Mutex<Option<usize>>> = Arc::default();
9396 let waiter = {
9397 let ctx = ctx.clone();
9398 let calls = Arc::clone(&recorder.calls);
9399 let resolved = Arc::clone(&resolved);
9400 std::thread::spawn(move || {
9401 let mut wait =
9402 Box::pin(ctx.wait_for_record("d", |s| *s == CheckpointStatus::Succeeded));
9403 let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
9404 while poll_rx.recv().is_ok() {
9405 let ready = wait.as_mut().poll(&mut cx).is_ready();
9406 if ready {
9407 *resolved.lock().unwrap() = Some(calls.lock().unwrap().len());
9408 }
9409 reply_tx.send(()).unwrap();
9410 if ready {
9411 return;
9412 }
9413 }
9414 })
9415 };
9416 poll_tx.send(()).unwrap();
9417 reply_rx.recv().unwrap();
9418 assert_eq!(
9419 *resolved.lock().unwrap(),
9420 None,
9421 "{case}: the wait registers"
9422 );
9423 let reply_rx = Arc::new(Mutex::new(reply_rx));
9424 {
9425 let poll_tx = poll_tx.clone();
9426 let reply_rx = Arc::clone(&reply_rx);
9427 ctx.shared_plugins()
9428 .expect("the context has plugins")
9429 .set_before_push(move |_| {
9430 poll_tx.send(()).unwrap();
9431 let _ = reply_rx
9433 .lock()
9434 .unwrap()
9435 .recv_timeout(Duration::from_millis(200));
9436 });
9437 }
9438 let write = ctx.checkpoint_updates_direct(vec![start("d")]);
9440 let _ = tokio::time::timeout(Duration::from_millis(500), write).await;
9441 for _ in 0..50 {
9442 if resolved.lock().unwrap().is_some() || poll_tx.send(()).is_err() {
9443 break;
9444 }
9445 let _ = reply_rx
9446 .lock()
9447 .unwrap()
9448 .recv_timeout(Duration::from_millis(100));
9449 }
9450 drop(poll_tx);
9451 waiter.join().unwrap();
9452 assert_eq!(
9453 *resolved.lock().unwrap(),
9454 Some(1),
9455 "{case}: the change is delivered before the wait resolves"
9456 );
9457 }
9458 }
9459}
9460
9461#[cfg(test)]
9464mod withdrawal_write_tests {
9465 use super::{CheckpointCoalescer, DurableContext, WriteError};
9466 use crate::client::{
9467 CheckpointOutput, ExecutionClient, ExecutionClientError, ExecutionClientErrorScope,
9468 GetStateOutput,
9469 };
9470 use crate::coordinator::PermitKind;
9471 use crate::driver::{FatalError, FatalSeverity};
9472 use crate::engine::CheckpointLog;
9473 use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
9474 use std::sync::Arc;
9475 use std::sync::atomic::{AtomicUsize, Ordering};
9476 use std::time::Duration;
9477
9478 #[derive(Default)]
9480 struct Tokenless(AtomicUsize);
9481
9482 impl ExecutionClient for Tokenless {
9483 fn checkpoint(
9484 &self,
9485 _execution_arn: &str,
9486 _checkpoint_token: &str,
9487 _updates: Vec<OperationUpdate>,
9488 ) -> std::pin::Pin<
9489 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
9490 > {
9491 self.0.fetch_add(1, Ordering::SeqCst);
9492 Box::pin(async { Ok(CheckpointOutput::new(None, Vec::new(), None)) })
9493 }
9494
9495 fn get_state(
9496 &self,
9497 _execution_arn: &str,
9498 _checkpoint_token: &str,
9499 _marker: Option<&str>,
9500 ) -> std::pin::Pin<
9501 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
9502 > {
9503 Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
9504 }
9505 }
9506
9507 fn update(id: &str, action: OperationAction) -> OperationUpdate {
9508 OperationUpdate::builder()
9509 .id(id)
9510 .r#type(OperationType::Step)
9511 .action(action)
9512 .build()
9513 .unwrap()
9514 }
9515
9516 fn ctx_with(client: Arc<dyn ExecutionClient>) -> DurableContext {
9518 DurableContext::new_root_with_client(
9519 "arn:test".to_owned(),
9520 lambda_runtime::Context::default(),
9521 Arc::new(CheckpointLog::empty()),
9522 client,
9523 "t0".to_owned(),
9524 )
9525 }
9526
9527 fn ctx_on(client: &Arc<Tokenless>, coalescer: Option<CheckpointCoalescer>) -> DurableContext {
9528 let client = Arc::clone(client) as Arc<dyn ExecutionClient>;
9529 let log = Arc::new(CheckpointLog::empty());
9530 match coalescer {
9531 Some(coalescer) => DurableContext::new_root_with_client_and_coalescer(
9532 "arn:test".to_owned(),
9533 lambda_runtime::Context::default(),
9534 log,
9535 client,
9536 "t0".to_owned(),
9537 coalescer,
9538 ),
9539 None => DurableContext::new_root_with_client(
9540 "arn:test".to_owned(),
9541 lambda_runtime::Context::default(),
9542 log,
9543 client,
9544 "t0".to_owned(),
9545 ),
9546 }
9547 }
9548
9549 #[tokio::test]
9552 async fn write_after_withdrawal_sends_nothing() {
9553 let client = Arc::new(Tokenless::default());
9554 let ctx = ctx_on(&client, None);
9555 let first = ctx.checkpoint_updates_after_drive(vec![update("a", OperationAction::Start)]);
9556 assert!(matches!(first.await, Err(WriteError::Stopped)));
9557 assert!(ctx.coordinator().withdrawn());
9558 assert_eq!(client.0.load(Ordering::SeqCst), 1);
9559 let second = ctx.checkpoint_updates_after_drive(vec![update("b", OperationAction::Start)]);
9560 assert!(matches!(second.await, Err(WriteError::Stopped)));
9561 assert_eq!(
9562 client.0.load(Ordering::SeqCst),
9563 1,
9564 "the second request is not sent"
9565 );
9566 }
9567
9568 #[derive(Default)]
9571 struct FailFirst(AtomicUsize);
9572
9573 impl ExecutionClient for FailFirst {
9574 fn checkpoint(
9575 &self,
9576 _execution_arn: &str,
9577 _checkpoint_token: &str,
9578 _updates: Vec<OperationUpdate>,
9579 ) -> std::pin::Pin<
9580 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
9581 > {
9582 let call = self.0.fetch_add(1, Ordering::SeqCst);
9583 Box::pin(async move {
9584 if call == 0 {
9585 Err(ExecutionClientError::new(
9586 ExecutionClientErrorScope::Execution,
9587 "down",
9588 ))
9589 } else {
9590 Ok(CheckpointOutput::new(
9591 Some(format!("t{call}")),
9592 Vec::new(),
9593 None,
9594 ))
9595 }
9596 })
9597 }
9598
9599 fn get_state(
9600 &self,
9601 _execution_arn: &str,
9602 _checkpoint_token: &str,
9603 _marker: Option<&str>,
9604 ) -> std::pin::Pin<
9605 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
9606 > {
9607 Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
9608 }
9609 }
9610
9611 async fn until(done: impl Fn() -> bool) {
9613 for _ in 0..1000 {
9614 if done() {
9615 return;
9616 }
9617 tokio::task::yield_now().await;
9618 }
9619 assert!(done(), "the condition never held");
9620 }
9621
9622 async fn failed_first_write() -> (DurableContext, Arc<FailFirst>, super::SubmittedWrite) {
9625 let client = Arc::new(FailFirst::default());
9626 let ctx = ctx_with(Arc::clone(&client) as Arc<dyn ExecutionClient>);
9627 let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9628 until(|| ctx.coordinator().write_failed()).await;
9629 (ctx, client, write)
9630 }
9631
9632 #[tokio::test(start_paused = true)]
9635 async fn failed_write_records_and_never_resolves() {
9636 let (ctx, client, write) = failed_first_write().await;
9637 let fatal = ctx.coordinator().fatal_error().expect("recorded");
9638 assert_eq!(
9639 fatal.error_type,
9640 crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
9641 );
9642 assert_eq!(
9643 ctx.coordinator().write_failure().map(|f| f.error_message),
9644 Some(fatal.error_message)
9645 );
9646 assert!(
9647 tokio::time::timeout(Duration::from_secs(60), write)
9648 .await
9649 .is_err(),
9650 "the failed write never resolves"
9651 );
9652 assert_eq!(client.0.load(Ordering::SeqCst), 1);
9653 }
9654
9655 #[tokio::test(start_paused = true)]
9658 async fn reserved_write_after_failure_sends_nothing() {
9659 let (ctx, client, _failed) = failed_first_write().await;
9660 let reserved = ctx.reserve_checkpoint_updates(vec![update("b", OperationAction::Start)]);
9661 let write = reserved.release();
9662 assert!(
9663 tokio::time::timeout(Duration::from_secs(60), write)
9664 .await
9665 .is_err(),
9666 "the reserved write never resolves"
9667 );
9668 assert_eq!(
9669 client.0.load(Ordering::SeqCst),
9670 1,
9671 "the reserved write is not sent"
9672 );
9673 }
9674
9675 #[tokio::test(start_paused = true)]
9678 async fn write_after_drive_after_failure_sends_nothing() {
9679 let (ctx, client, _failed) = failed_first_write().await;
9680 ctx.coordinator().finish();
9681 let written = ctx
9682 .checkpoint_updates_after_drive(vec![update("b", OperationAction::Start)])
9683 .await;
9684 assert!(matches!(written, Err(WriteError::Stopped)), "{written:?}");
9685 assert_eq!(client.0.load(Ordering::SeqCst), 1, "nothing is sent");
9686 }
9687
9688 #[tokio::test(start_paused = true)]
9691 async fn write_without_client_records_the_failure() {
9692 let ctx = DurableContext::test_context();
9693 let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9694 assert!(
9695 tokio::time::timeout(Duration::from_secs(60), write)
9696 .await
9697 .is_err(),
9698 "the write never resolves"
9699 );
9700 let fatal = ctx.coordinator().write_failure().expect("latched");
9701 assert_eq!(
9702 fatal.error_type,
9703 crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
9704 );
9705 assert!(
9706 fatal
9707 .error_message
9708 .contains("no execution client configured"),
9709 "{fatal:?}"
9710 );
9711 }
9712
9713 #[tokio::test(start_paused = true)]
9717 async fn write_without_client_records_under_the_token_lock() {
9718 let ctx = DurableContext::test_context();
9719 let held = ctx.inner.checkpoint_token.lock().await;
9720 let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9721 tokio::time::sleep(Duration::from_secs(60)).await;
9722 assert!(
9723 !ctx.coordinator().write_failed(),
9724 "nothing is recorded while the lock is held"
9725 );
9726 assert!(ctx.coordinator().fatal_error().is_none());
9727 drop(held);
9728 until(|| ctx.coordinator().write_failed()).await;
9729 drop(write);
9730 }
9731
9732 #[tokio::test(start_paused = true)]
9736 async fn write_without_client_after_withdrawal_records_nothing() {
9737 let ctx = DurableContext::test_context();
9738 ctx.coordinator().withdraw("arn:test", "req", || {});
9739 let written = ctx
9740 .checkpoint_direct_with_events(vec![update("a", OperationAction::Start)], &[], None)
9741 .await;
9742 assert!(matches!(written, Err(WriteError::Stopped)), "{written:?}");
9743 assert!(!ctx.coordinator().write_failed());
9744 assert!(ctx.coordinator().fatal_error().is_none());
9745 }
9746
9747 #[tokio::test(start_paused = true)]
9751 async fn failed_write_drops_the_open_batch_and_rejects_later_joins() {
9752 let client = Arc::new(FailFirst::default());
9753 let ctx = DurableContext::new_root_with_client_and_coalescer(
9754 "arn:test".to_owned(),
9755 lambda_runtime::Context::default(),
9756 Arc::new(CheckpointLog::empty()),
9757 Arc::clone(&client) as Arc<dyn ExecutionClient>,
9758 "t0".to_owned(),
9759 CheckpointCoalescer::new(Duration::from_secs(60)),
9760 );
9761 let buffered = ctx.submit_checkpoint_updates(vec![update("b", OperationAction::Start)]);
9762 let failed = ctx
9763 .checkpoint_updates_after_drive(vec![update("a", OperationAction::Start)])
9764 .await;
9765 assert!(matches!(failed, Err(WriteError::Failed(_))), "{failed:?}");
9766 assert!(ctx.coordinator().write_failed());
9767 assert!(
9768 ctx.queued_update_ids().is_empty(),
9769 "the open batch is dropped"
9770 );
9771 let later = ctx.submit_checkpoint_updates(vec![update("c", OperationAction::Start)]);
9772 assert!(
9773 ctx.queued_update_ids().is_empty(),
9774 "the later join is rejected"
9775 );
9776 ctx.flush_pending_checkpoints().await;
9777 let all = async { tokio::join!(buffered, later) };
9778 assert!(
9779 tokio::time::timeout(Duration::from_secs(120), all)
9780 .await
9781 .is_err(),
9782 "no contributor resolves"
9783 );
9784 assert_eq!(client.0.load(Ordering::SeqCst), 1, "nothing more is sent");
9785 }
9786
9787 trait Held {}
9789 impl<T> Held for T {}
9790
9791 fn submission_racing_a_failed_write(
9800 coalescer: CheckpointCoalescer,
9801 point: crate::coordinator::race_points::RacePoint,
9802 lock_queue: fn(&DurableContext) -> Box<dyn Held + '_>,
9803 ) {
9804 let runtime = tokio::runtime::Builder::new_current_thread()
9805 .enable_all()
9806 .build()
9807 .unwrap();
9808 let client = Arc::new(FailFirst::default());
9809 let ctx = {
9810 let _entered = runtime.enter();
9811 let client = Arc::clone(&client) as Arc<dyn ExecutionClient>;
9812 DurableContext::new_root_with_client_and_coalescer(
9813 "arn:test".to_owned(),
9814 lambda_runtime::Context::default(),
9815 Arc::new(CheckpointLog::empty()),
9816 client,
9817 "t0".to_owned(),
9818 coalescer,
9819 )
9820 };
9821 let held = lock_queue(&ctx);
9822 let (reached_tx, reached_rx) = std::sync::mpsc::channel();
9823 let submitter = {
9824 let ctx = ctx.clone();
9825 let handle = runtime.handle().clone();
9826 std::thread::spawn(move || {
9827 let _entered = handle.enter();
9828 crate::coordinator::race_points::set(move |reached| {
9829 if reached == point {
9830 let _ = reached_tx.send(());
9831 }
9832 });
9833 drop(ctx.submit_checkpoint_updates(vec![update("b", OperationAction::Start)]));
9834 })
9835 };
9836 reached_rx.recv().unwrap();
9839 ctx.coordinator().fail_write(
9840 FatalError {
9841 error_type: "Failed".to_owned(),
9842 error_message: "failed".to_owned(),
9843 severity: FatalSeverity::FailExecution,
9844 },
9845 || {},
9846 );
9847 drop(held);
9848 submitter.join().unwrap();
9849 assert!(
9850 ctx.queued_update_ids().is_empty(),
9851 "nothing waits to be sent"
9852 );
9853 assert_eq!(client.0.load(Ordering::SeqCst), 0, "nothing is sent");
9854 }
9855
9856 #[test]
9859 fn buffered_submission_waiting_at_the_buffer_lock_sees_the_latch() {
9860 submission_racing_a_failed_write(
9861 CheckpointCoalescer::new(Duration::from_secs(60)),
9862 crate::coordinator::race_points::RacePoint::BufferLock,
9863 |ctx| Box::new(ctx.inner.coalescer.hold_buffer_lock()),
9864 );
9865 }
9866
9867 #[tokio::test(start_paused = true)]
9870 async fn join_after_a_failed_write_is_refused_its_permit() {
9871 let client = Arc::new(FailFirst::default());
9872 let ctx = DurableContext::new_root_with_client_and_coalescer(
9873 "arn:test".to_owned(),
9874 lambda_runtime::Context::default(),
9875 Arc::new(CheckpointLog::empty()),
9876 Arc::clone(&client) as Arc<dyn ExecutionClient>,
9877 "t0".to_owned(),
9878 CheckpointCoalescer::new(Duration::from_secs(60)),
9879 );
9880 ctx.coordinator().fail_write(
9881 FatalError {
9882 error_type: "Failed".to_owned(),
9883 error_message: "failed".to_owned(),
9884 severity: FatalSeverity::FailExecution,
9885 },
9886 || {},
9887 );
9888 assert!(
9889 ctx.coordinator().acquire(PermitKind::Checkpoint).is_none(),
9890 "no checkpoint permit"
9891 );
9892 assert!(
9893 ctx.coordinator().acquire(PermitKind::Activity).is_some(),
9894 "other permits are still taken"
9895 );
9896 drop(ctx.submit_checkpoint_updates(vec![update("b", OperationAction::Start)]));
9897 assert_eq!(ctx.queued_update_ids(), Vec::<String>::new());
9898 }
9899
9900 #[tokio::test(start_paused = true)]
9903 async fn end_flush_stops_at_withdrawal() {
9904 let client = Arc::new(Tokenless::default());
9905 let ctx = ctx_on(
9906 &client,
9907 Some(CheckpointCoalescer::new(Duration::from_secs(60))),
9908 );
9909 let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9910 ctx.flush_pending_checkpoints().await;
9911 assert!(ctx.coordinator().withdrawn());
9912 assert_eq!(client.0.load(Ordering::SeqCst), 1);
9913 assert!(
9914 tokio::time::timeout(Duration::from_secs(120), write)
9915 .await
9916 .is_err(),
9917 "the contributor never resolves"
9918 );
9919 assert_eq!(
9920 client.0.load(Ordering::SeqCst),
9921 1,
9922 "the window flush sends nothing"
9923 );
9924 }
9925}