1use std::future::Future;
13use std::time::Duration;
14
15use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
16
17use crate::builders::{Backoff, JitterStrategy, RetryStrategyConfig};
18use crate::context::{DurableContext, StepContext, SubmittedWrite};
19use crate::engine::{CheckpointStatus, OperationId};
20use crate::error::{
21 OperationError, OperationErrorKind, STEP_INTERRUPTED_MESSAGE, StepError, StepErrorKind,
22};
23use crate::plugin::UserBody;
24use crate::serdes::SerdesContext;
25use crate::start::StartStep;
26use crate::tracing_layer;
27use crate::{BoxError, OperationSubType, RetryDecision, RetryStrategy, Serdes};
28
29pub(crate) fn default_retry_strategy() -> RetryStrategy {
37 RetryStrategyConfig::default_preset().into_retry_strategy()
38}
39
40impl RetryStrategyConfig {
41 pub(crate) fn decide(&self, err: &StepError, attempt: u32) -> RetryDecision {
58 if attempt >= self.max_attempts() {
59 return RetryDecision::Stop;
60 }
61 if let Some(predicate) = self.retry_predicate()
62 && !predicate(err)
63 {
64 return RetryDecision::Stop;
65 }
66
67 let initial = self.initial_delay().as_secs_f64();
69 let base = match self.backoff() {
70 Backoff::Exponential { rate } => {
72 let exponent = (i32::try_from(attempt).unwrap_or(1)) - 1;
73 initial * rate.powi(exponent)
74 }
75 Backoff::Linear { increment } => {
77 initial + increment.as_secs_f64() * f64::from(attempt.saturating_sub(1))
78 }
79 }
80 .min(self.max_delay().as_secs_f64());
81
82 let jittered = match self.jitter() {
83 JitterStrategy::None => base,
84 JitterStrategy::Half => base / 2.0 + rand_full_jitter(base / 2.0),
86 JitterStrategy::Full => rand_full_jitter(base),
88 };
89
90 RetryDecision::retry(quantize_delay(jittered))
91 }
92
93 pub(crate) fn into_retry_strategy(self) -> RetryStrategy {
97 Box::new(move |err: &StepError, attempt: u32| self.decide(err, attempt))
98 }
99}
100
101pub(crate) fn rand_full_jitter(max_secs: f64) -> f64 {
106 use std::collections::hash_map::DefaultHasher;
107 use std::hash::{Hash, Hasher};
108 use std::time::SystemTime;
109
110 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
111
112 let mut hasher = DefaultHasher::new();
113 SystemTime::now().hash(&mut hasher);
114 std::thread::current().id().hash(&mut hasher);
115 COUNTER
117 .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
118 .hash(&mut hasher);
119 let bits = hasher.finish();
120 #[expect(clippy::cast_precision_loss)] let fraction = (bits as f64) / (u64::MAX as f64);
123 fraction * max_secs
124}
125
126pub(crate) fn quantize_delay(secs: f64) -> Duration {
134 Duration::try_from_secs_f64(secs.round().max(1.0)).unwrap_or(Duration::from_secs(1))
135}
136
137#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
162#[non_exhaustive]
163pub enum StepSemantics {
164 #[default]
176 AtLeastOncePerRetry,
177
178 AtMostOncePerRetry,
191}
192
193pub(crate) const STEP_SUB_TYPE: &str = "Step";
195
196pub(crate) struct StepExecution<O, F, S> {
204 pub(crate) ctx: DurableContext,
205 pub(crate) op_id: OperationId,
206 pub(crate) name: Option<String>,
207 pub(crate) retry_strategy: Option<RetryStrategy>,
208 pub(crate) serdes: S,
209 pub(crate) semantics: StepSemantics,
210 pub(crate) sub_type: OperationSubType,
212 pub(crate) plugin_name: Option<String>,
216 pub(crate) closure: F,
217 pub(crate) _marker: std::marker::PhantomData<fn() -> O>,
218}
219
220impl<O, F, Fut, S> StepExecution<O, F, S>
221where
222 O: Send + 'static,
223 F: FnOnce(StepContext) -> Fut + Clone + Send + 'static,
224 Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
225 S: Serdes<O>,
226{
227 pub(crate) async fn execute(self) -> Result<O, OperationError> {
239 let (core, closure) = self.split();
240 Self::run(core, closure, None).await
241 }
242
243 pub(crate) fn execute_started(
247 self,
248 ) -> impl Future<Output = Result<O, OperationError>> + Send + 'static {
249 let (core, closure) = self.split();
250 let first = core.start();
251 Self::run(core, closure, Some(first))
252 }
253
254 fn split(self) -> (StepCore<O, S>, F) {
256 let Self {
257 ctx,
258 op_id,
259 name,
260 retry_strategy,
261 serdes,
262 semantics,
263 sub_type,
264 plugin_name,
265 closure,
266 _marker,
267 } = self;
268 let core = StepCore {
269 ctx,
270 op_id,
271 name,
272 retry_strategy,
273 serdes,
274 semantics,
275 sub_type,
276 plugin_name,
277 _marker: std::marker::PhantomData,
278 };
279 (core, closure)
280 }
281
282 async fn run(
285 core: StepCore<O, S>,
286 mut closure: F,
287 mut first: Option<Result<StartStep, OperationError>>,
288 ) -> Result<O, OperationError> {
289 let mut resumed = false;
293 loop {
294 match core.before(resumed, first.take()).await? {
295 StepPrelude::Done(result) => return result,
296 StepPrelude::Wait => {}
297 StepPrelude::Run {
298 attempt,
299 span,
300 start,
301 info,
302 } => {
303 let body = UserBody::Step(core.op_id.positional().to_owned());
309 let step_ctx = StepContext::new(attempt);
310 let closure = &mut closure;
314 let ran = core
320 .ctx
321 .hold_activity(crate::plugin::run_user_function(
322 core.ctx.shared_plugins(),
323 info,
324 body,
325 span,
326 move || {
327 let attempt_closure = (*closure).clone();
328 attempt_closure(step_ctx)
329 },
330 |error: &BoxError| {
331 crate::error::wire_error_for(&**error, STEP_FALLBACK_ERROR_TYPE)
332 },
333 ))
334 .await;
335 core.await_start(start).await?;
338 match core.settle(attempt, ran).await? {
339 StepSettled::Done(value) => return Ok(value),
340 StepSettled::Retry => {}
341 }
342 }
343 }
344 core.ctx
354 .wait_for_record(core.op_id.wire(), |status| {
355 *status == CheckpointStatus::Ready || status.is_terminal()
356 })
357 .await;
358 resumed = true;
359 }
360 }
361}
362
363struct StepCore<O, S> {
369 ctx: DurableContext,
370 op_id: OperationId,
371 name: Option<String>,
372 retry_strategy: Option<RetryStrategy>,
373 serdes: S,
374 semantics: StepSemantics,
375 sub_type: OperationSubType,
376 plugin_name: Option<String>,
378 _marker: std::marker::PhantomData<fn() -> O>,
379}
380
381enum StepPrelude<O> {
383 Done(Result<O, OperationError>),
386 Wait,
389 Run {
391 attempt: u32,
393 span: tracing::Span,
395 start: Option<SubmittedWrite>,
400 info: std::sync::Arc<crate::plugin::OperationInfo>,
402 },
403}
404
405enum StepSettled<O> {
407 Done(O),
409 Retry,
411}
412
413impl<O, S> StepCore<O, S>
414where
415 O: Send + 'static,
416 S: Serdes<O>,
417{
418 fn emit_replayed(&self, resumed: bool, attempt: u32) {
421 if !resumed {
422 self.ctx.emit_operation_replayed(
423 self.op_id.wire(),
424 self.name.as_deref(),
425 "Step",
426 Some(self.sub_type.as_str()),
427 attempt,
428 );
429 }
430 }
431
432 fn start_update(&self) -> OperationUpdate {
434 build_start_update(
435 self.op_id.wire(),
436 self.name.as_deref(),
437 self.sub_type.as_str(),
438 self.ctx.parent_wire_id(),
439 )
440 }
441
442 async fn before(
453 &self,
454 resumed: bool,
455 first: Option<Result<StartStep, OperationError>>,
456 ) -> Result<StepPrelude<O>, OperationError> {
457 let started = match first {
460 Some(started) => started,
461 None => self.start(),
462 }?;
463
464 let wire_id = self.op_id.wire();
465
466 self.after_start(resumed, started, wire_id).await
469 }
470
471 fn start(&self) -> Result<StartStep, OperationError> {
473 crate::start::start_step(
474 &self.ctx,
475 &crate::start::StartSpec {
476 op_id: &self.op_id,
477 operation_type: "Step",
478 sub_type: self.sub_type.as_str(),
479 name: self.name.as_deref(),
480 order: match self.semantics {
481 StepSemantics::AtLeastOncePerRetry => crate::start::StartOrder::QueueThenHook,
482 StepSemantics::AtMostOncePerRetry => crate::start::StartOrder::AwaitResponse,
483 },
484 },
485 |view| self.cycle(view),
486 || self.start_update(),
487 |plugins, is_replay| self.operation_started(plugins, is_replay),
488 )
489 }
490
491 async fn after_start(
493 &self,
494 resumed: bool,
495 started: StartStep,
496 wire_id: &str,
497 ) -> Result<StepPrelude<O>, OperationError> {
498 let start = match started {
499 StartStep::Refused => return std::future::pending().await,
500 StartStep::Queued(write) => Some(write),
501 StartStep::Recorded(view)
505 if view.status == CheckpointStatus::Started
506 && self.semantics == StepSemantics::AtLeastOncePerRetry =>
507 {
508 None
509 }
510 StartStep::Recorded(view) => return self.recorded(resumed, &view).await,
511 };
512
513 let attempt = self.ctx.get_attempt(&self.op_id).saturating_add(1);
517
518 let start = match start {
523 None => None,
524 Some(start) if self.semantics == StepSemantics::AtLeastOncePerRetry => Some(start),
525 Some(start) => {
526 if let Err(err) = start.await {
527 return self.ctx.checkpoint_failure_unrecoverable(err).await;
528 }
529 if let Some(plugins) = self.ctx.plugins() {
530 self.operation_started(plugins, attempt > 1);
531 }
532 None
533 }
534 };
535
536 let is_replay = false; let span = tracing_layer::operation_span(
540 self.ctx.execution_arn(),
541 &self.ctx.lambda_context().request_id,
542 self.ctx.lambda_context().tenant_id.as_deref(),
543 wire_id,
544 self.name.as_deref(),
545 attempt,
546 is_replay,
547 );
548
549 let info = self.attempt_started(attempt);
550
551 Ok(StepPrelude::Run {
552 attempt,
553 span,
554 start,
555 info,
556 })
557 }
558
559 async fn recorded(
564 &self,
565 resumed: bool,
566 view: &crate::engine::CheckpointStatusView,
567 ) -> Result<StepPrelude<O>, OperationError> {
568 let positional_id = self.op_id.positional();
569 let wire_id = self.op_id.wire();
570 match view.status {
571 CheckpointStatus::Succeeded => {
572 let serdes_ctx = SerdesContext::new(wire_id, self.ctx.execution_arn());
577 let payload = self.ctx.checkpoint_result_payload(positional_id);
578 let value = replay_success(
579 &self.ctx,
580 self.name.as_deref(),
581 &self.serdes,
582 payload,
583 serdes_ctx,
584 )
585 .await?;
586 self.emit_replayed(resumed, view.attempt);
587 Ok(StepPrelude::Done(Ok(value)))
588 }
589 CheckpointStatus::Failed => {
590 self.emit_replayed(resumed, view.attempt);
591 let wire = self
592 .ctx
593 .checkpoint_wire_error(positional_id)
594 .unwrap_or_default();
595 Ok(StepPrelude::Done(Err(replay_failure(wire, wire_id))))
596 }
597 CheckpointStatus::Started => {
598 let attempt = self.ctx.get_attempt(&self.op_id).saturating_add(1);
603 handle_failure(
604 &self.ctx,
605 wire_id,
606 self.name.as_deref(),
607 self.sub_type.as_str(),
608 self.plugin_name.as_deref(),
609 self.retry_strategy.as_ref(),
610 interrupted_failure(),
611 attempt,
612 )
613 .await?;
614 Ok(StepPrelude::Wait)
615 }
616 CheckpointStatus::Unknown(ref raw) => {
617 Err(self.ctx.unrecognized_status_error(wire_id, raw))
622 }
623 CheckpointStatus::Pending
627 | CheckpointStatus::Ready
628 | CheckpointStatus::Cancelled
629 | CheckpointStatus::TimedOut
630 | CheckpointStatus::Stopped => Ok(StepPrelude::Wait),
631 }
632 }
633
634 fn cycle(&self, view: &crate::engine::CheckpointStatusView) -> crate::start::Cycle {
637 step_cycle(self.semantics, view)
638 }
639
640 fn operation_started(&self, plugins: &crate::plugin::InvocationPlugins, is_replay: bool) {
642 let info = step_info(
643 &self.ctx,
644 self.op_id.wire(),
645 self.sub_type.as_str(),
646 self.name.as_deref(),
647 self.plugin_name.as_deref(),
648 )
649 .with_status(crate::OperationStatus::Started)
650 .with_replay(is_replay);
651 crate::plugin::operation_started(plugins, &info);
652 }
653
654 fn attempt_started(&self, attempt: u32) -> std::sync::Arc<crate::plugin::OperationInfo> {
661 let info = step_info(
662 &self.ctx,
663 self.op_id.wire(),
664 self.sub_type.as_str(),
665 self.name.as_deref(),
666 self.plugin_name.as_deref(),
667 );
668 match self.ctx.plugins() {
669 Some(plugins) => crate::plugin::attempt_start_only(plugins, info, attempt),
670 None => std::sync::Arc::new(info),
671 }
672 }
673
674 async fn await_start(&self, start: Option<SubmittedWrite>) -> Result<(), OperationError> {
681 if let Some(start) = start
682 && let Err(err) = start.await
683 {
684 return self.ctx.checkpoint_failure_unrecoverable(err).await;
685 }
686 Ok(())
687 }
688
689 async fn settle(
692 &self,
693 attempt: u32,
694 ran: crate::plugin::Ran<O>,
695 ) -> Result<StepSettled<O>, OperationError> {
696 let wire_id = self.op_id.wire();
697 match ran.into_parts() {
698 Ok(value) => handle_success(
699 &self.ctx,
700 wire_id,
701 self.name.as_deref(),
702 self.sub_type.as_str(),
703 self.plugin_name.as_deref(),
704 &self.serdes,
705 value,
706 attempt,
707 )
708 .await
709 .map(StepSettled::Done),
710 Err((err, wire)) => {
711 handle_failure(
712 &self.ctx,
713 wire_id,
714 self.name.as_deref(),
715 self.sub_type.as_str(),
716 self.plugin_name.as_deref(),
717 self.retry_strategy.as_ref(),
718 (StepError::execution_failed(err), wire),
719 attempt,
720 )
721 .await?;
722 Ok(StepSettled::Retry)
723 }
724 }
725 }
726}
727
728pub(crate) fn step_cycle(
736 semantics: StepSemantics,
737 view: &crate::engine::CheckpointStatusView,
738) -> crate::start::Cycle {
739 use crate::start::Cycle;
740 match view.status {
741 CheckpointStatus::Ready
742 | CheckpointStatus::Cancelled
743 | CheckpointStatus::TimedOut
744 | CheckpointStatus::Stopped => Cycle::Start {
745 is_replay: view.attempt > 0,
746 },
747 CheckpointStatus::Started if semantics == StepSemantics::AtLeastOncePerRetry => {
748 Cycle::Resume
749 }
750 _ => Cycle::Record,
751 }
752}
753
754fn step_info(
757 ctx: &DurableContext,
758 wire_id: &str,
759 sub_type: &str,
760 name: Option<&str>,
761 plugin_name: Option<&str>,
762) -> crate::plugin::OperationInfo {
763 let info = crate::plugin::OperationInfo::for_operation(
764 ctx,
765 wire_id,
766 crate::OperationKind::Step,
767 sub_type,
768 name,
769 );
770 match plugin_name {
771 Some(plugin_name) => info.with_name(Some(plugin_name.to_owned())),
772 None => info,
773 }
774}
775
776#[expect(clippy::too_many_arguments)] async fn handle_success<O, S: Serdes<O>>(
783 ctx: &DurableContext,
784 wire_id: &str,
785 name: Option<&str>,
786 sub_type: &str,
787 plugin_name: Option<&str>,
788 serdes: &S,
789 value: O,
790 attempt: u32,
791) -> Result<O, OperationError> {
792 let serdes_ctx = SerdesContext::new(wire_id, ctx.execution_arn());
793
794 let serialized = match serialize_value(ctx, name, serdes, value, serdes_ctx.clone()).await {
803 Ok(serialized) => serialized,
804 Err(op_err) => {
805 let wire = crate::error::serialization_failure_wire(&op_err);
810 let update = build_fail_update(wire_id, name, sub_type, ctx.parent_wire_id(), &wire);
811 if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
812 return ctx.checkpoint_failure_unrecoverable(client_err).await;
813 }
814 return Err(op_err
815 .with_operation(wire_id, CheckpointStatus::Failed.wire_str())
816 .with_wire(wire));
817 }
818 };
819
820 let update = build_succeed_update(wire_id, name, sub_type, ctx.parent_wire_id(), &serialized);
822 if let Err(err) = ctx.checkpoint_updates(vec![update]).await {
823 return ctx.checkpoint_failure_unrecoverable(err).await;
824 }
825
826 if let Some(plugins) = ctx.plugins() {
827 crate::plugin::attempt_ended(
828 plugins,
829 step_info(ctx, wire_id, sub_type, name, plugin_name)
830 .with_status(crate::OperationStatus::Succeeded)
831 .with_result(Some(serialized.clone())),
832 crate::plugin::AttemptEnd {
833 attempt,
834 outcome: crate::plugin::AttemptOutcome::Succeeded,
835 error: None,
836 ends_operation: true,
837 },
838 );
839 }
840
841 deserialize_result(ctx, name, serdes, serialized, serdes_ctx).await
843}
844
845#[expect(clippy::too_many_arguments)] async fn handle_failure(
857 ctx: &DurableContext,
858 wire_id: &str,
859 name: Option<&str>,
860 sub_type: &str,
861 plugin_name: Option<&str>,
862 retry_strategy: Option<&RetryStrategy>,
863 (step_err, wire): (StepError, crate::error::WireError),
864 attempt: u32,
865) -> Result<(), OperationError> {
866 let plugins = ctx
872 .plugins()
873 .filter(|_| !matches!(step_err.kind(), StepErrorKind::Interrupted));
874 let attempt_ended = |status: crate::OperationStatus, ends_operation: bool| {
876 if let Some(plugins) = plugins {
877 crate::plugin::attempt_ended(
878 plugins,
879 step_info(ctx, wire_id, sub_type, name, plugin_name)
880 .with_status(status)
881 .with_error(Some(wire.clone())),
882 crate::plugin::AttemptEnd {
883 attempt,
884 outcome: crate::plugin::AttemptOutcome::Failed,
885 error: Some(wire.clone()),
886 ends_operation,
887 },
888 );
889 }
890 };
891
892 let decision = if let Some(strategy) = retry_strategy {
895 strategy(&step_err, attempt)
896 } else {
897 default_retry_strategy()(&step_err, attempt)
898 };
899
900 match decision {
901 RetryDecision::Retry { delay } => {
902 let whole_secs = delay
906 .as_secs()
907 .saturating_add(u64::from(delay.subsec_nanos() > 0));
908 let delay_secs = i32::try_from(whole_secs.max(1)).unwrap_or(i32::MAX);
909 let update = build_retry_update(
910 wire_id,
911 name,
912 sub_type,
913 ctx.parent_wire_id(),
914 &wire,
915 delay_secs,
916 );
917 if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
918 return ctx.checkpoint_failure_unrecoverable(client_err).await;
919 }
920 attempt_ended(crate::OperationStatus::Pending, false);
921 Ok(())
922 }
923 RetryDecision::Stop => {
924 let update = build_fail_update(wire_id, name, sub_type, ctx.parent_wire_id(), &wire);
926 if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
927 return ctx.checkpoint_failure_unrecoverable(client_err).await;
928 }
929 attempt_ended(crate::OperationStatus::Failed, true);
930
931 Err(
935 OperationError::from_kind(OperationErrorKind::Step(step_err))
936 .with_operation(wire_id, CheckpointStatus::Failed.wire_str())
937 .with_wire(wire),
938 )
939 }
940 }
941}
942
943fn interrupted_failure() -> (StepError, crate::error::WireError) {
946 let message: BoxError = STEP_INTERRUPTED_MESSAGE.into();
947 let wire = crate::error::wire_error_for(&*message, STEP_INTERRUPTED_ERROR_TYPE);
948 (StepError::interrupted(), wire)
949}
950
951fn build_start_update(
954 wire_id: &str,
955 name: Option<&str>,
956 sub_type: &str,
957 parent_wire_id: Option<&str>,
958) -> OperationUpdate {
959 let mut builder = OperationUpdate::builder()
960 .id(wire_id)
961 .r#type(OperationType::Step)
962 .sub_type(sub_type)
963 .action(OperationAction::Start);
964
965 if let Some(n) = name {
966 builder = builder.name(n);
967 }
968
969 if let Some(parent) = parent_wire_id {
970 builder = builder.parent_id(parent);
971 }
972
973 #[expect(clippy::expect_used)] builder
976 .build()
977 .expect("all required OperationUpdate fields set")
978}
979
980fn build_succeed_update(
981 wire_id: &str,
982 name: Option<&str>,
983 sub_type: &str,
984 parent_wire_id: Option<&str>,
985 payload: &str,
986) -> OperationUpdate {
987 let mut builder = OperationUpdate::builder()
988 .id(wire_id)
989 .r#type(OperationType::Step)
990 .sub_type(sub_type)
991 .action(OperationAction::Succeed)
992 .payload(payload);
993
994 if let Some(n) = name {
995 builder = builder.name(n);
996 }
997
998 if let Some(parent) = parent_wire_id {
999 builder = builder.parent_id(parent);
1000 }
1001
1002 #[expect(clippy::expect_used)] builder
1004 .build()
1005 .expect("all required OperationUpdate fields set")
1006}
1007
1008fn build_retry_update(
1009 wire_id: &str,
1010 name: Option<&str>,
1011 sub_type: &str,
1012 parent_wire_id: Option<&str>,
1013 error: &crate::error::WireError,
1014 delay_secs: i32,
1015) -> OperationUpdate {
1016 let mut builder = OperationUpdate::builder()
1017 .id(wire_id)
1018 .r#type(OperationType::Step)
1019 .sub_type(sub_type)
1020 .action(OperationAction::Retry)
1021 .error(error.to_error_object())
1022 .step_options(
1023 aws_sdk_lambda::types::StepOptions::builder()
1024 .next_attempt_delay_seconds(delay_secs)
1025 .build(),
1026 );
1027
1028 if let Some(n) = name {
1029 builder = builder.name(n);
1030 }
1031
1032 if let Some(parent) = parent_wire_id {
1033 builder = builder.parent_id(parent);
1034 }
1035
1036 #[expect(clippy::expect_used)] builder
1038 .build()
1039 .expect("all required OperationUpdate fields set")
1040}
1041
1042fn build_fail_update(
1043 wire_id: &str,
1044 name: Option<&str>,
1045 sub_type: &str,
1046 parent_wire_id: Option<&str>,
1047 error: &crate::error::WireError,
1048) -> OperationUpdate {
1049 let mut builder = OperationUpdate::builder()
1050 .id(wire_id)
1051 .r#type(OperationType::Step)
1052 .sub_type(sub_type)
1053 .action(OperationAction::Fail)
1054 .error(error.to_error_object());
1055
1056 if let Some(n) = name {
1057 builder = builder.name(n);
1058 }
1059
1060 if let Some(parent) = parent_wire_id {
1061 builder = builder.parent_id(parent);
1062 }
1063
1064 #[expect(clippy::expect_used)] builder
1066 .build()
1067 .expect("all required OperationUpdate fields set")
1068}
1069
1070async fn serialize_value<O, S: Serdes<O>>(
1083 ctx: &DurableContext,
1084 name: Option<&str>,
1085 serdes: &S,
1086 value: O,
1087 serdes_ctx: SerdesContext,
1088) -> Result<String, OperationError> {
1089 let operation_id = serdes_ctx.operation_id().to_owned();
1090 match ctx.hold_activity(serdes.serialize(value, serdes_ctx)).await {
1091 Ok(serialized) => Ok(serialized),
1092 Err(err) => Err(step_serialization_error(
1093 ctx.route_serdes_error("serialize", &operation_id, name, err)
1094 .await,
1095 )),
1096 }
1097}
1098
1099async fn deserialize_result<O, S: Serdes<O>>(
1105 ctx: &DurableContext,
1106 name: Option<&str>,
1107 serdes: &S,
1108 serialized: String,
1109 serdes_ctx: SerdesContext,
1110) -> Result<O, OperationError> {
1111 let operation_id = serdes_ctx.operation_id().to_owned();
1112 match ctx
1113 .hold_activity(serdes.deserialize(serialized, serdes_ctx))
1114 .await
1115 {
1116 Ok(value) => Ok(value),
1117 Err(err) => Err(step_serialization_error(
1118 ctx.route_serdes_error("deserialize", &operation_id, name, err)
1119 .await,
1120 )),
1121 }
1122}
1123
1124fn step_serialization_error(e: BoxError) -> OperationError {
1127 OperationError::from_kind(OperationErrorKind::Step(StepError::new(
1128 StepErrorKind::SerializationFailed,
1129 Some(e),
1130 )))
1131}
1132
1133async fn replay_success<O, S: Serdes<O>>(
1134 ctx: &DurableContext,
1135 name: Option<&str>,
1136 serdes: &S,
1137 result: Option<String>,
1138 serdes_ctx: SerdesContext,
1139) -> Result<O, OperationError> {
1140 let payload = result.unwrap_or_else(|| "null".to_owned());
1141 deserialize_result(ctx, name, serdes, payload, serdes_ctx).await
1142}
1143
1144fn replay_failure(wire: crate::error::WireError, wire_id: &str) -> OperationError {
1165 let kind = match wire.error_type() {
1166 Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE) => StepErrorKind::SerializationFailed,
1167 Some(STEP_INTERRUPTED_ERROR_TYPE) => StepErrorKind::Interrupted,
1168 _ => StepErrorKind::ExecutionFailed,
1169 };
1170 OperationError::from_kind(OperationErrorKind::Step(StepError::new(
1171 kind,
1172 Some(crate::error::ReplayedFailure::source_from(wire.clone())),
1173 )))
1174 .with_operation(wire_id, CheckpointStatus::Failed.wire_str())
1175 .with_wire(wire)
1176}
1177
1178const STEP_FALLBACK_ERROR_TYPE: &str = "Error";
1190
1191const STEP_INTERRUPTED_ERROR_TYPE: &str = "StepInterruptedError";
1194
1195#[cfg(test)]
1196#[expect(clippy::panic)]
1197#[expect(clippy::disallowed_methods)]
1201mod tests {
1202 use super::*;
1203 use crate::context::DurableContext;
1204 use crate::engine::CheckpointLog;
1205 use crate::plugin::recorder::UfHooks;
1206 use std::sync::Arc;
1207
1208 #[tokio::test]
1211 async fn replay_success_deserializes_json() {
1212 let payload = "42".to_owned();
1213 let ctx = SerdesContext::new("op-1", "arn:test");
1214 let (dctx, _client) = context_over(Vec::new());
1215 let result: Result<i32, OperationError> =
1216 replay_success(&dctx, None, &crate::serdes::JsonSerdes, Some(payload), ctx).await;
1217 assert_eq!(result.unwrap(), 42);
1218 }
1219
1220 #[tokio::test]
1221 async fn replay_success_null_returns_unit() {
1222 let ctx = SerdesContext::new("op-1", "arn:test");
1223 let (dctx, _client) = context_over(Vec::new());
1224 let result: Result<(), OperationError> =
1225 replay_success(&dctx, None, &crate::serdes::JsonSerdes, None, ctx).await;
1226 assert!(result.is_ok());
1227 }
1228
1229 #[test]
1230 fn replay_failure_keeps_kind_and_wire_fields_apart() {
1231 let wire = crate::error::WireError::new(Some("TypeError"), Some("bad input"));
1232 let err = replay_failure(wire, "wire-1");
1233 let OperationErrorKind::Step(step_err) = err.kind() else {
1237 unreachable!("replay builds a step error");
1238 };
1239 let StepErrorKind::ExecutionFailed = step_err.kind() else {
1240 unreachable!("ordinary failures replay as ExecutionFailed");
1241 };
1242 assert_eq!(err.to_string(), "operation error: step");
1244 let source = std::error::Error::source(step_err).expect("synthetic source");
1246 let replayed = source
1247 .downcast_ref::<crate::error::ReplayedFailure>()
1248 .expect("replay source is a ReplayedFailure");
1249 assert_eq!(replayed.error_type(), Some("TypeError"));
1250 assert_eq!(replayed.error_message(), Some("bad input"));
1251 assert_eq!(err.operation_id(), Some("wire-1"));
1253 assert_eq!(err.status(), Some("FAILED"));
1254 assert_eq!(
1255 err.wire().and_then(crate::error::WireError::error_type),
1256 Some("TypeError")
1257 );
1258 let chain = format!("{err:#}");
1261 assert!(chain.contains("bad input"), "got: {chain}");
1262 assert!(
1263 !chain.contains("TypeError"),
1264 "type must not fold into text: {chain}"
1265 );
1266 }
1267
1268 #[test]
1269 fn replay_failure_empty_record_displays_unknown() {
1270 let err = replay_failure(crate::error::WireError::default(), "wire-1");
1271 let chain = format!("{err:#}");
1272 assert!(chain.contains("unknown error"), "got: {chain}");
1273 }
1274
1275 #[test]
1276 fn replay_failure_serialization_discriminator_keeps_its_kind() {
1277 let wire = crate::error::WireError::new(
1278 Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE),
1279 Some("bad payload"),
1280 );
1281 let err = replay_failure(wire, "wire-1");
1282 let OperationErrorKind::Step(step_err) = err.kind() else {
1283 unreachable!("replay builds a step error");
1284 };
1285 assert!(matches!(
1286 step_err.kind(),
1287 StepErrorKind::SerializationFailed
1288 ));
1289 }
1290
1291 #[tokio::test]
1294 async fn serialize_deserialize_round_trip() {
1295 let ctx = SerdesContext::new("op-1", "arn:test");
1296 let (dctx, _client) = context_over(Vec::new());
1297 let serialized = serialize_value(
1298 &dctx,
1299 None,
1300 &crate::serdes::JsonSerdes,
1301 "hello".to_owned(),
1302 ctx.clone(),
1303 )
1304 .await
1305 .unwrap();
1306 let deserialized: String =
1307 deserialize_result(&dctx, None, &crate::serdes::JsonSerdes, serialized, ctx)
1308 .await
1309 .unwrap();
1310 assert_eq!(deserialized, "hello");
1311 }
1312
1313 #[tokio::test]
1314 async fn serialize_with_custom_serdes_uppercases() {
1315 struct Upper;
1316 impl Serdes<String> for Upper {
1317 #[expect(clippy::unused_async_trait_impl)]
1319 async fn serialize(
1320 &self,
1321 value: String,
1322 _context: SerdesContext,
1323 ) -> Result<String, BoxError> {
1324 Ok(serde_json::to_string(&value)?.to_uppercase())
1325 }
1326 #[expect(clippy::unused_async_trait_impl)]
1328 async fn deserialize(
1329 &self,
1330 wire: String,
1331 _context: SerdesContext,
1332 ) -> Result<String, BoxError> {
1333 Ok(serde_json::from_str(&wire)?)
1334 }
1335 }
1336 let ctx = SerdesContext::new("op-1", "arn:test");
1337 let (dctx, _client) = context_over(Vec::new());
1338 let result = serialize_value(&dctx, None, &Upper, "hello".to_owned(), ctx)
1339 .await
1340 .unwrap();
1341 assert_eq!(result, "\"HELLO\"");
1342 }
1343
1344 #[test]
1347 fn default_retry_stops_at_max_attempts() {
1348 let strategy = default_retry_strategy();
1349 let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
1350 let decision = strategy(&err, 6);
1352 assert_eq!(decision, RetryDecision::Stop);
1353 }
1354
1355 #[test]
1356 fn default_retry_retries_below_max() {
1357 let strategy = default_retry_strategy();
1358 let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
1359 let decision = strategy(&err, 1);
1360 match decision {
1361 RetryDecision::Retry { delay } => {
1362 assert!(delay.as_secs() >= 1);
1364 assert!(delay.as_secs() <= 5);
1365 }
1366 RetryDecision::Stop => panic!("expected retry for attempt 1"),
1367 }
1368 }
1369
1370 #[test]
1371 fn default_retry_delay_grows_with_attempts() {
1372 let strategy = default_retry_strategy();
1375 let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
1376 let decision = strategy(&err, 5);
1377 match decision {
1378 RetryDecision::Retry { delay } => {
1379 assert!(delay.as_secs() <= 60);
1381 assert!(delay.as_secs() >= 1);
1382 }
1383 RetryDecision::Stop => panic!("expected retry for attempt 5"),
1384 }
1385 }
1386
1387 #[tokio::test]
1390 async fn step_live_success_checkpoints_and_returns() {
1391 use crate::client::InMemoryExecutionClient;
1392
1393 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
1394 let log = Arc::new(CheckpointLog::empty());
1395 let ctx = DurableContext::new_root_with_client(
1396 "arn:test".to_owned(),
1397 lambda_runtime::Context::default(),
1398 log,
1399 client,
1400 "token0".to_owned(),
1401 );
1402 let op_id = ctx.mint_id();
1403 let closure = |_: StepContext| async { Ok::<i32, BoxError>(42) };
1404
1405 let exec = StepExecution {
1406 ctx,
1407 op_id,
1408 name: Some("test-step".to_owned()),
1409 retry_strategy: None,
1410 serdes: crate::serdes::JsonSerdes,
1411 semantics: StepSemantics::default(),
1412 sub_type: OperationSubType::STEP,
1413 plugin_name: None,
1414 closure,
1415 _marker: std::marker::PhantomData,
1416 };
1417
1418 let result = exec.execute().await;
1419 assert_eq!(result.unwrap(), 42);
1420 }
1421
1422 #[tokio::test]
1423 async fn per_operation_serdes_serializes_step_result() {
1424 use crate::client::InMemoryExecutionClient;
1425
1426 struct Upper;
1430 impl Serdes<String> for Upper {
1431 #[expect(clippy::unused_async_trait_impl)]
1433 async fn serialize(
1434 &self,
1435 value: String,
1436 _context: SerdesContext,
1437 ) -> Result<String, BoxError> {
1438 Ok(serde_json::to_string(&value)?.to_uppercase())
1439 }
1440 #[expect(clippy::unused_async_trait_impl)]
1442 async fn deserialize(
1443 &self,
1444 wire: String,
1445 _context: SerdesContext,
1446 ) -> Result<String, BoxError> {
1447 Ok(serde_json::from_str(&wire.to_lowercase())?)
1448 }
1449 }
1450
1451 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
1452 let log = Arc::new(CheckpointLog::empty());
1453 let ctx = DurableContext::new_root_with_client_and_defaults(
1454 "arn:test".to_owned(),
1455 lambda_runtime::Context::default(),
1456 log,
1457 client.clone(),
1458 "token0".to_owned(),
1459 Duration::ZERO,
1460 0,
1461 None,
1462 );
1463 let op_id = ctx.mint_id();
1464 let closure = |_: StepContext| async { Ok::<String, BoxError>("hello".to_owned()) };
1465
1466 let exec = StepExecution {
1467 ctx,
1468 op_id,
1469 name: Some("greet".to_owned()),
1470 retry_strategy: None,
1471 serdes: Upper,
1472 semantics: StepSemantics::default(),
1473 sub_type: OperationSubType::STEP,
1474 plugin_name: None,
1475 closure,
1476 _marker: std::marker::PhantomData,
1477 };
1478
1479 let result = exec.execute().await;
1481 assert_eq!(result.unwrap(), "hello");
1482
1483 let updates = client.recorded_updates();
1487 let succeed = updates
1488 .iter()
1489 .find(|u| matches!(u.action(), OperationAction::Succeed))
1490 .expect("a Succeed update must be recorded");
1491 assert_eq!(
1492 succeed.payload(),
1493 Some("\"HELLO\""),
1494 "the per-operation serdes must serialize the step result"
1495 );
1496 }
1497
1498 #[tokio::test]
1499 async fn step_replay_returns_frozen_result_without_re_executing() {
1500 use crate::engine::{CheckpointLog, CheckpointRecord};
1501 use std::sync::atomic::{AtomicBool, Ordering};
1502
1503 let executed = Arc::new(AtomicBool::new(false));
1504 let executed_clone = Arc::clone(&executed);
1505
1506 let wire_key = crate::engine::compute_wire_id_public("1");
1510 let record = CheckpointRecord {
1511 id: wire_key.clone(),
1512 status: CheckpointStatus::Succeeded,
1513 result: Some("99".to_owned()),
1514 error_type: None,
1515 error_message: None,
1516 error_data: None,
1517 stack_trace: None,
1518 attempt: 0,
1519 invoke_result: None,
1520 invoke_error_type: None,
1521 invoke_error_message: None,
1522 invoke_error_data: None,
1523 invoke_stack_trace: None,
1524 replay_children: false,
1525 callback_id: None,
1526 op_type: None,
1527 sub_type: None,
1528 due_at: None,
1529 op_name: None,
1530 parent_id: None,
1531 start_timestamp: None,
1532 end_timestamp: None,
1533 };
1534 let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
1535 let ctx = DurableContext::new_root(
1536 "arn:test".to_owned(),
1537 lambda_runtime::Context::default(),
1538 log,
1539 );
1540 let op_id = ctx.mint_id(); let closure = move |_: StepContext| {
1543 executed_clone.store(true, Ordering::SeqCst);
1544 async { Ok::<i32, BoxError>(0) }
1545 };
1546
1547 let exec = StepExecution {
1548 ctx,
1549 op_id,
1550 name: None,
1551 retry_strategy: None,
1552 serdes: crate::serdes::JsonSerdes,
1553 semantics: StepSemantics::default(),
1554 sub_type: OperationSubType::STEP,
1555 plugin_name: None,
1556 closure,
1557 _marker: std::marker::PhantomData,
1558 };
1559
1560 let result = exec.execute().await;
1561 assert_eq!(result.unwrap(), 99);
1562 assert!(
1563 !executed.load(Ordering::SeqCst),
1564 "closure should NOT execute during replay"
1565 );
1566 }
1567
1568 #[tokio::test]
1569 async fn step_replay_failure_returns_error_without_re_executing() {
1570 use crate::engine::{CheckpointLog, CheckpointRecord};
1571 use std::sync::atomic::{AtomicBool, Ordering};
1572
1573 let executed = Arc::new(AtomicBool::new(false));
1574 let executed_clone = Arc::clone(&executed);
1575
1576 let wire_key = crate::engine::compute_wire_id_public("1");
1577 let record = CheckpointRecord {
1578 id: wire_key.clone(),
1579 status: CheckpointStatus::Failed,
1580 result: None,
1581 error_type: Some("CustomError".to_owned()),
1582 error_message: Some("it broke".to_owned()),
1583 error_data: None,
1584 stack_trace: None,
1585 attempt: 0,
1586 invoke_result: None,
1587 invoke_error_type: None,
1588 invoke_error_message: None,
1589 invoke_error_data: None,
1590 invoke_stack_trace: None,
1591 replay_children: false,
1592 callback_id: None,
1593 op_type: None,
1594 sub_type: None,
1595 due_at: None,
1596 op_name: None,
1597 parent_id: None,
1598 start_timestamp: None,
1599 end_timestamp: None,
1600 };
1601 let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
1602 let ctx = DurableContext::new_root(
1603 "arn:test".to_owned(),
1604 lambda_runtime::Context::default(),
1605 log,
1606 );
1607 let op_id = ctx.mint_id();
1608
1609 let closure = move |_: StepContext| {
1610 executed_clone.store(true, Ordering::SeqCst);
1611 async { Ok::<String, BoxError>("should not run".to_owned()) }
1612 };
1613
1614 let exec = StepExecution {
1615 ctx,
1616 op_id,
1617 name: None,
1618 retry_strategy: None,
1619 serdes: crate::serdes::JsonSerdes,
1620 semantics: StepSemantics::default(),
1621 sub_type: OperationSubType::STEP,
1622 plugin_name: None,
1623 closure,
1624 _marker: std::marker::PhantomData,
1625 };
1626
1627 let result = exec.execute().await;
1628 assert!(result.is_err());
1629 let err = result.unwrap_err();
1630 let err_msg = format!("{err:#}");
1631 assert!(err_msg.contains("it broke"), "got: {err_msg}");
1632 assert_eq!(
1634 err.wire().and_then(crate::error::WireError::error_type),
1635 Some("CustomError")
1636 );
1637 assert!(
1638 !executed.load(Ordering::SeqCst),
1639 "closure should NOT execute during replay"
1640 );
1641 }
1642
1643 #[tokio::test]
1644 async fn step_retry_then_stop_on_exhaustion() {
1645 use crate::client::InMemoryExecutionClient;
1646
1647 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
1648 let log = Arc::new(CheckpointLog::empty());
1649 let ctx = DurableContext::new_root_with_client(
1650 "arn:test".to_owned(),
1651 lambda_runtime::Context::default(),
1652 log,
1653 client,
1654 "token0".to_owned(),
1655 );
1656 let op_id = ctx.mint_id();
1657
1658 let no_retry: RetryStrategy =
1660 Box::new(|_err: &StepError, _attempt: u32| RetryDecision::Stop);
1661
1662 let closure = |_: StepContext| async { Err::<i32, BoxError>("always fails".into()) };
1663
1664 let exec = StepExecution {
1665 ctx,
1666 op_id,
1667 name: None,
1668 retry_strategy: Some(no_retry),
1669 serdes: crate::serdes::JsonSerdes,
1670 semantics: StepSemantics::default(),
1671 sub_type: OperationSubType::STEP,
1672 plugin_name: None,
1673 closure,
1674 _marker: std::marker::PhantomData,
1675 };
1676
1677 let result = exec.execute().await;
1678 assert!(result.is_err());
1679 let err_msg = format!("{:#}", result.unwrap_err());
1680 assert!(
1681 err_msg.contains("retries exhausted") || err_msg.contains("always fails"),
1682 "got: {err_msg}"
1683 );
1684 }
1685
1686 fn context_with_plugin(
1689 hooks: &crate::plugin::test_support::OpHooks,
1690 window: Duration,
1691 ) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
1692 let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
1693 let ctx = DurableContext::new_root_with_client_and_defaults(
1694 "arn:test".to_owned(),
1695 lambda_runtime::Context::default(),
1696 Arc::new(CheckpointLog::empty()),
1697 Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
1698 "token0".to_owned(),
1699 window,
1700 0,
1701 Some(hooks.plugins()),
1702 );
1703 (ctx, client)
1704 }
1705
1706 #[tokio::test]
1707 async fn hooks_run_outside_the_token_lock() {
1708 let lock: Arc<std::sync::OnceLock<Arc<tokio::sync::Mutex<String>>>> = Arc::default();
1709 let probe_lock = Arc::clone(&lock);
1710 let hooks = crate::plugin::test_support::OpHooks::probing(move || {
1711 let free = probe_lock
1712 .get()
1713 .is_some_and(|mutex| mutex.try_lock().is_ok());
1714 free.to_string()
1715 });
1716 let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
1717 lock.set(ctx.checkpoint_token_mutex()).expect("set once");
1718 let value = ctx.step(|_| async { Ok(1_u32) }).name("a").await.unwrap();
1719 assert_eq!(value, 1);
1720 let probes: Vec<String> = hooks.records().into_iter().map(|r| r.probe).collect();
1721 assert_eq!(probes, ["true"; 4]);
1722 }
1723
1724 #[tokio::test]
1725 async fn hooks_run_on_the_operation_task() {
1726 let hooks = crate::plugin::test_support::OpHooks::probing(|| {
1727 format!("{:?}", tokio::task::try_id())
1728 });
1729 let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
1730 let body_task = Arc::new(std::sync::Mutex::new(None));
1731 let seen = Arc::clone(&body_task);
1732 let value = ctx
1733 .step(move |_| async move {
1734 *seen.lock().unwrap() = tokio::task::try_id();
1735 Ok(1_u32)
1736 })
1737 .name("a")
1738 .spawn()
1739 .await
1740 .unwrap();
1741 assert_eq!(value, 1);
1742 let body = body_task.lock().unwrap().expect("the body ran on a task");
1743 let records = hooks.records();
1744 let probe_of = |hook: &str| {
1745 records
1746 .iter()
1747 .find(|r| r.hook == hook)
1748 .map(|r| r.probe.clone())
1749 .expect("the hook ran")
1750 };
1751 let expected = format!("{:?}", Some(body));
1752 assert_eq!(probe_of("attempt_end"), expected);
1753 assert_eq!(probe_of("end"), expected);
1754 }
1755
1756 #[tokio::test]
1757 async fn no_plugins_keeps_checkpoint_updates() {
1758 let (ctx, client) = context_over(Vec::new());
1759 assert!(ctx.plugins().is_none());
1760 let value = ctx.step(|_| async { Ok(1_u32) }).name("a").await.unwrap();
1761 assert_eq!(value, 1);
1762 let updates = client.recorded_updates();
1763 let wire_id = crate::engine::compute_wire_id_public("1");
1764 let fields: Vec<_> = updates
1765 .iter()
1766 .map(|u| {
1767 (
1768 u.id().to_owned(),
1769 u.r#type().clone(),
1770 u.sub_type().map(str::to_owned),
1771 u.name().map(str::to_owned),
1772 u.action().clone(),
1773 u.payload().map(str::to_owned),
1774 u.parent_id().map(str::to_owned),
1775 u.error().is_some(),
1776 )
1777 })
1778 .collect();
1779 let step = |action, payload: Option<&str>| {
1780 (
1781 wire_id.clone(),
1782 OperationType::Step,
1783 Some(STEP_SUB_TYPE.to_owned()),
1784 Some("a".to_owned()),
1785 action,
1786 payload.map(str::to_owned),
1787 None,
1788 false,
1789 )
1790 };
1791 assert_eq!(
1792 fields,
1793 [
1794 step(OperationAction::Start, None),
1795 step(OperationAction::Succeed, Some("1")),
1796 ]
1797 );
1798 }
1799
1800 #[tokio::test]
1803 async fn failed_step_end_infos_carry_the_written_error() {
1804 let hooks = crate::plugin::test_support::OpHooks::default();
1805 let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
1806 let err = ctx
1807 .step(|_| async { Err::<u32, BoxError>("boom".into()) })
1808 .name("a")
1809 .retry_strategy(|_err, _attempt| RetryDecision::Stop)
1810 .await
1811 .unwrap_err();
1812 assert!(err.wire().is_some());
1813 let records = hooks.records();
1814 let ends: Vec<_> = records
1815 .iter()
1816 .filter(|r| r.hook == "attempt_end" || r.hook == "end")
1817 .collect();
1818 assert_eq!(ends.len(), 2);
1819 let attempt_end = ends.first().expect("an attempt end");
1820 assert_eq!(attempt_end.error.as_deref(), Some("boom"));
1821 assert_eq!(attempt_end.status, Some(crate::OperationStatus::Failed));
1822 }
1823
1824 fn uf_context(
1828 hooks: &UfHooks,
1829 records: Vec<(String, crate::engine::CheckpointRecord)>,
1830 ) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
1831 let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
1832 let ctx = DurableContext::new_root_with_client_and_defaults(
1833 "arn:test".to_owned(),
1834 lambda_runtime::Context::default(),
1835 Arc::new(CheckpointLog::from_records(records)),
1836 Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
1837 "token0".to_owned(),
1838 Duration::ZERO,
1839 0,
1840 Some(hooks.plugins()),
1841 );
1842 (ctx, client)
1843 }
1844
1845 async fn drive<F>(ctx: &DurableContext, fut: F) -> crate::driver::InvocationOutcome
1847 where
1848 F: IntoFuture + Send + 'static,
1849 F::IntoFuture: Send + 'static,
1850 {
1851 let signal = Arc::clone(ctx.coordinator());
1852 let ownership = Arc::clone(ctx.task_ownership());
1853 crate::driver::test_support::outcome_of(ctx.clone(), ownership, signal, fut).await
1854 }
1855
1856 fn update_of(
1858 client: &crate::client::InMemoryExecutionClient,
1859 name: &str,
1860 action: &OperationAction,
1861 ) -> Option<OperationUpdate> {
1862 client
1863 .recorded_updates()
1864 .into_iter()
1865 .find(|u| u.name() == Some(name) && u.action() == action)
1866 }
1867
1868 fn type_and_message(wire: &crate::error::WireError) -> (Option<String>, Option<String>) {
1870 (
1871 wire.error_type().map(str::to_owned),
1872 wire.error_message().map(str::to_owned),
1873 )
1874 }
1875
1876 fn update_error(update: &OperationUpdate) -> (Option<String>, Option<String>) {
1878 let error = update.error().expect("the update carries an error object");
1879 (
1880 error.error_type().map(str::to_owned),
1881 error.error_message().map(str::to_owned),
1882 )
1883 }
1884
1885 const LIVE_ATTEMPT: [&str; 6] = [
1887 "attempt_start",
1888 "uf_start",
1889 "wrap_enter",
1890 "wrap_exit",
1891 "uf_end",
1892 "attempt_end",
1893 ];
1894
1895 #[tokio::test]
1899 async fn interrupted_attempt_fires_no_user_function_hook() {
1900 use std::sync::atomic::{AtomicU32, Ordering};
1901 let runs = Arc::new(AtomicU32::new(0));
1902 let step = |ctx: &DurableContext, runs: &Arc<AtomicU32>| {
1903 let runs = Arc::clone(runs);
1904 ctx.step(move |_| async move {
1905 runs.fetch_add(1, Ordering::SeqCst);
1906 Ok(42_u32)
1907 })
1908 .name("s")
1909 .semantics(StepSemantics::AtMostOncePerRetry)
1910 .retry_strategy(|_err, _attempt| RetryDecision::retry(Duration::from_secs(1)))
1911 };
1912
1913 let hooks = UfHooks::default();
1915 let (ctx, client) = uf_context(
1916 &hooks,
1917 vec![step_record(CheckpointStatus::Started, 0, None)],
1918 );
1919 let outcome = drive(&ctx, step(&ctx, &runs)).await;
1920 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
1921 assert_eq!(runs.load(Ordering::SeqCst), 0, "the body did not run");
1922 let retry = recorded_error_for(&client, &OperationAction::Retry);
1923 assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
1924 assert!(!hooks.any_user_function_record(), "{:?}", hooks.records());
1925
1926 let hooks = UfHooks::default();
1928 let (ctx, _client) =
1929 uf_context(&hooks, vec![step_record(CheckpointStatus::Ready, 1, None)]);
1930 assert_eq!(step(&ctx, &runs).await.unwrap(), 42);
1931 let mut expected = vec!["start"];
1932 expected.extend(LIVE_ATTEMPT);
1933 expected.push("end");
1934 assert_eq!(hooks.hooks_of("s"), expected);
1935 assert_eq!(hooks.find("s", "uf_start").info_attempt, Some(2));
1936 assert_eq!(hooks.find("s", "uf_end").ok, Some(true));
1937 }
1938
1939 #[tokio::test]
1942 async fn retry_wait_fires_no_user_function_hook() {
1943 use std::sync::atomic::{AtomicU32, Ordering};
1944 let runs = Arc::new(AtomicU32::new(0));
1945 let step = |ctx: &DurableContext, runs: &Arc<AtomicU32>| {
1946 let runs = Arc::clone(runs);
1947 ctx.step(move |_| async move {
1948 runs.fetch_add(1, Ordering::SeqCst);
1949 Err::<u32, BoxError>("transient".into())
1950 })
1951 .name("s")
1952 .retry_strategy(|_err, _attempt| RetryDecision::retry(Duration::from_secs(10)))
1953 };
1954
1955 let hooks = UfHooks::default();
1957 let (ctx, client) = uf_context(&hooks, Vec::new());
1958 let outcome = drive(&ctx, step(&ctx, &runs)).await;
1959 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
1960 assert_eq!(runs.load(Ordering::SeqCst), 1);
1961 let mut expected = vec!["start"];
1962 expected.extend(LIVE_ATTEMPT);
1963 assert_eq!(hooks.hooks_of("s"), expected);
1964 assert_eq!(hooks.find("s", "uf_end").ok, Some(false));
1965 assert!(update_of(&client, "s", &OperationAction::Retry).is_some());
1966
1967 let hooks = UfHooks::default();
1969 let (wire_key, mut record) = step_record(CheckpointStatus::Pending, 1, None);
1970 record.due_at = Some(std::time::SystemTime::now() + Duration::from_secs(10));
1971 let (ctx, client) = uf_context(&hooks, vec![(wire_key, record)]);
1972 let outcome = drive(&ctx, step(&ctx, &runs)).await;
1973 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
1974 assert_eq!(
1975 runs.load(Ordering::SeqCst),
1976 1,
1977 "no body ran in invocation 2"
1978 );
1979 assert!(!hooks.any_user_function_record(), "{:?}", hooks.records());
1980 assert!(
1981 client.recorded_updates().is_empty(),
1982 "the step stays PENDING: {:?}",
1983 client.recorded_updates()
1984 );
1985 }
1986
1987 #[tokio::test]
1990 async fn end_hook_before_outcome_record() {
1991 let client_cell: Arc<std::sync::OnceLock<Arc<crate::client::InMemoryExecutionClient>>> =
1992 Arc::default();
1993 let probe_client = Arc::clone(&client_cell);
1994 let hooks = UfHooks::probing(move || {
1995 let actions: Vec<String> = probe_client
1996 .get()
1997 .map(|client| client.recorded_updates())
1998 .unwrap_or_default()
1999 .iter()
2000 .map(|u| format!("{:?}", u.action()))
2001 .collect();
2002 actions.join(",")
2003 });
2004 let (ctx, client) = uf_context(&hooks, Vec::new());
2005 client_cell.set(Arc::clone(&client)).expect("set once");
2006 let body_client = Arc::clone(&client);
2009 let value = ctx
2010 .step(move |_| async move {
2011 for _ in 0..1000 {
2012 if update_of(&body_client, "s", &OperationAction::Start).is_some() {
2013 return Ok(1_u32);
2014 }
2015 tokio::task::yield_now().await;
2016 }
2017 Err("the START was never written".into())
2018 })
2019 .name("s")
2020 .await
2021 .unwrap();
2022 assert_eq!(value, 1);
2023 assert_eq!(hooks.find("s", "uf_end").probe, "Start");
2024 assert!(update_of(&client, "s", &OperationAction::Succeed).is_some());
2025 }
2026
2027 #[tokio::test]
2032 async fn failed_body_wire_derived_once() {
2033 use crate::plugin::recorder::Counted;
2034
2035 let hooks = UfHooks::default();
2037 let (ctx, client) = uf_context(&hooks, Vec::new());
2038 let dc = Counted::new();
2039 let body_dc = dc.clone();
2040 let err = ctx
2041 .step(move |_| async move { Err::<u32, BoxError>(body_dc.boxed()) })
2042 .name("step")
2043 .retry_strategy(|_err, _attempt| RetryDecision::Stop)
2044 .await
2045 .unwrap_err();
2046 drop(err);
2047 assert_eq!(dc.calls(), 1, "step");
2048 let end = hooks.find("step", "uf_end").wire.expect("an Err outcome");
2049 let attempt_end = hooks.find("step", "attempt_end").wire.expect("an error");
2050 assert_eq!(type_and_message(&end), type_and_message(&attempt_end));
2051 let fail = update_of(&client, "step", &OperationAction::Fail).expect("a FAIL");
2052 assert_eq!(type_and_message(&end), update_error(&fail));
2053 assert_eq!(end.error_message(), Some("counted"));
2054
2055 let hooks = UfHooks::default();
2057 let (ctx, client) = uf_context(&hooks, Vec::new());
2058 let dc = Counted::new();
2059 let body_dc = dc.clone();
2060 let err = ctx
2061 .run_in_child_context(
2062 move |_child| async move { Err::<u32, BoxError>(body_dc.boxed()) },
2063 )
2064 .name("child")
2065 .await
2066 .unwrap_err();
2067 drop(err);
2068 assert_eq!(dc.calls(), 1, "child");
2069 let end = hooks.find("child", "uf_end").wire.expect("an Err outcome");
2070 let fail = update_of(&client, "child", &OperationAction::Fail).expect("a FAIL");
2071 assert_eq!(type_and_message(&end), update_error(&fail));
2072
2073 let dc = Counted::new();
2076 let probe_dc = dc.clone();
2077 let hooks = UfHooks::probing(move || probe_dc.calls().to_string());
2078 let (ctx, client) = uf_context(&hooks, Vec::new());
2079 let body_dc = dc.clone();
2080 let err = ctx
2081 .wait_for_callback::<String>(move |_step_ctx, _callback_id| async move {
2082 Err(body_dc.boxed())
2083 })
2084 .name("cb")
2085 .submitter_retry(|_err, _attempt| RetryDecision::Stop)
2086 .await
2087 .unwrap_err();
2088 drop(err);
2089 assert_eq!(hooks.find("cb-submitter", "uf_end").probe, "1");
2091 assert_eq!(hooks.find("cb-submitter", "end").probe, "1");
2092 assert_eq!(dc.calls(), 1, "the submitter only");
2096 assert_eq!(hooks.find("cb", "uf_end").probe, "1");
2097 let outer = hooks.find("cb", "uf_end").wire.expect("an Err outcome");
2098 assert_eq!(outer.error_message(), Some("counted"));
2099 let end = hooks
2100 .find("cb-submitter", "uf_end")
2101 .wire
2102 .expect("an Err outcome");
2103 let attempt_end = hooks
2104 .find("cb-submitter", "attempt_end")
2105 .wire
2106 .expect("an error");
2107 assert_eq!(type_and_message(&end), type_and_message(&attempt_end));
2108 assert_eq!(end.error_message(), Some("counted"));
2109 let fail = client
2111 .recorded_updates()
2112 .into_iter()
2113 .find(|u| u.r#type() == &OperationType::Step && u.action() == &OperationAction::Fail)
2114 .expect("the submitter's FAIL");
2115 assert_eq!(type_and_message(&end), update_error(&fail));
2116
2117 map_item_wire_derived_once().await;
2119 }
2120
2121 #[tokio::test]
2124 async fn failed_body_records_a_propagated_record_unchanged() {
2125 let hooks = UfHooks::default();
2126 let (ctx, client) = uf_context(&hooks, Vec::new());
2127 let err = ctx
2128 .step(|_| async {
2129 let wire = crate::error::WireError::new(Some("InnerType"), Some("inner"))
2130 .with_error_data(Some("inner-data"))
2131 .with_stack_trace(vec!["inner-frame".to_owned()]);
2132 Err::<u32, BoxError>(Box::new(crate::error::recorded_step_error(wire)))
2133 })
2134 .name("step")
2135 .retry_strategy(|_err, _attempt| RetryDecision::Stop)
2136 .await
2137 .unwrap_err();
2138 drop(err);
2139 let fail = update_of(&client, "step", &OperationAction::Fail).expect("a FAIL");
2140 let error = fail.error().expect("the FAIL carries an error object");
2141 assert_eq!(error.error_type(), Some("InnerType"));
2142 assert_eq!(error.error_message(), Some("inner"));
2143 assert_eq!(error.error_data(), Some("inner-data"));
2144 assert_eq!(error.stack_trace(), ["inner-frame"]);
2145 }
2146
2147 async fn map_item_wire_derived_once() {
2150 use crate::builders::map_parallel::CompletionConfig;
2151 use crate::map_parallel::NestingMode;
2152 use crate::plugin::recorder::Counted;
2153
2154 for mode in [NestingMode::Normal, NestingMode::Flat] {
2155 let hooks = UfHooks::default();
2156 let (ctx, client) = uf_context(&hooks, Vec::new());
2157 let dc = Counted::new();
2158 let body_dc = dc.clone();
2159 let batch = ctx
2160 .map(vec![1_u32], move |_child, _item, _idx| {
2161 let dc = body_dc.clone();
2162 async move { Err::<u32, BoxError>(dc.boxed()) }
2163 })
2164 .name("m")
2165 .nesting(mode)
2166 .completion(CompletionConfig::with_tolerated_failure_count(1))
2167 .await_batch()
2168 .await
2169 .unwrap();
2170 assert_eq!(dc.calls(), 1, "{mode:?}");
2171 let end = hooks
2172 .find("map-item-0", "uf_end")
2173 .wire
2174 .expect("an Err outcome");
2175 let item = batch.errors().next().expect("the item failed");
2176 assert_eq!(
2177 type_and_message(&end),
2178 (
2179 item.error_type.map(str::to_owned),
2180 Some(item.message.to_owned())
2181 ),
2182 "{mode:?}"
2183 );
2184 let fail = update_of(&client, "map-item-0", &OperationAction::Fail);
2185 match mode {
2186 NestingMode::Normal => {
2187 assert_eq!(type_and_message(&end), update_error(&fail.expect("a FAIL")));
2188 }
2189 NestingMode::Flat => assert!(fail.is_none(), "a flat item writes no record"),
2190 }
2191 }
2192 }
2193
2194 #[tokio::test]
2199 async fn failing_step_display_calls_durable_op() {
2200 use crate::plugin::recorder::Counted;
2201 let hooks = UfHooks::default();
2202 let (ctx, client) = uf_context(&hooks, Vec::new());
2203 let claim_ctx = ctx.clone();
2204 let dc = Counted::calling(move || drop(claim_ctx.wait(Duration::from_secs(1))));
2205 let body_dc = dc.clone();
2206 let step = ctx
2207 .step(move |_| async move { Err::<u32, BoxError>(body_dc.boxed()) })
2208 .name("s")
2209 .retry_strategy(|_err, _attempt| RetryDecision::Stop);
2210 assert!(
2211 tokio::time::timeout(Duration::from_millis(50), step.into_future())
2212 .await
2213 .is_err(),
2214 "the step's FAIL is refused, so the step never resolves"
2215 );
2216 assert_eq!(dc.calls(), 1);
2217 assert!(
2218 update_of(&client, "s", &OperationAction::Fail).is_none(),
2219 "no terminal update is recorded"
2220 );
2221 assert!(update_of(&client, "s", &OperationAction::Succeed).is_none());
2222 let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
2223 assert!(
2224 fatal
2225 .error_message
2226 .contains("while the SDK records the outcome of operation \"1\""),
2227 "{}",
2228 fatal.error_message
2229 );
2230 }
2231
2232 struct FailingSerializer;
2234
2235 impl<T: Send + 'static> Serdes<T> for FailingSerializer {
2236 #[expect(clippy::unused_async_trait_impl)]
2238 async fn serialize(&self, _value: T, _context: SerdesContext) -> Result<String, BoxError> {
2239 Err("cannot encode".into())
2240 }
2241
2242 #[expect(clippy::unused_async_trait_impl)]
2244 async fn deserialize(&self, _wire: String, _context: SerdesContext) -> Result<T, BoxError> {
2245 Err("cannot decode".into())
2246 }
2247 }
2248
2249 #[tokio::test]
2252 async fn serialization_failure_keeps_its_own_wire() {
2253 let hooks = UfHooks::default();
2254 let (ctx, client) = uf_context(&hooks, Vec::new());
2255 let err = ctx
2256 .step(|_| async { Ok(1_u32) })
2257 .name("s")
2258 .serdes(FailingSerializer)
2259 .await
2260 .unwrap_err();
2261 drop(err);
2262 let end = hooks.find("s", "uf_end");
2263 assert_eq!(end.ok, Some(true));
2264 assert_eq!(end.wire, None);
2265 let fail = update_of(&client, "s", &OperationAction::Fail).expect("a FAIL");
2266 let (error_type, _) = update_error(&fail);
2267 assert_eq!(
2268 error_type.as_deref(),
2269 Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE)
2270 );
2271 }
2272
2273 fn step_record(
2275 status: CheckpointStatus,
2276 attempt: u32,
2277 error: Option<&crate::error::WireError>,
2278 ) -> (String, crate::engine::CheckpointRecord) {
2279 let wire_key = crate::engine::compute_wire_id_public("1");
2280 let record = crate::engine::CheckpointRecord {
2281 id: wire_key.clone(),
2282 status,
2283 result: None,
2284 error_type: error
2285 .and_then(crate::error::WireError::error_type)
2286 .map(str::to_owned),
2287 error_message: error
2288 .and_then(crate::error::WireError::error_message)
2289 .map(str::to_owned),
2290 error_data: error
2291 .and_then(crate::error::WireError::error_data)
2292 .map(str::to_owned),
2293 stack_trace: None,
2294 attempt,
2295 invoke_result: None,
2296 invoke_error_type: None,
2297 invoke_error_message: None,
2298 invoke_error_data: None,
2299 invoke_stack_trace: None,
2300 replay_children: false,
2301 callback_id: None,
2302 op_type: None,
2303 sub_type: None,
2304 due_at: None,
2305 op_name: None,
2306 parent_id: None,
2307 start_timestamp: None,
2308 end_timestamp: None,
2309 };
2310 (wire_key, record)
2311 }
2312
2313 fn context_over(
2316 records: Vec<(String, crate::engine::CheckpointRecord)>,
2317 ) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
2318 let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
2319 let ctx = DurableContext::new_root_with_client(
2320 "arn:test".to_owned(),
2321 lambda_runtime::Context::default(),
2322 Arc::new(CheckpointLog::from_records(records)),
2323 Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
2324 "token0".to_owned(),
2325 );
2326 (ctx, client)
2327 }
2328
2329 fn recorded_fail(client: &crate::client::InMemoryExecutionClient) -> crate::error::WireError {
2331 let updates = client.recorded_updates();
2332 let fail = updates
2333 .iter()
2334 .find(|u| matches!(u.action(), OperationAction::Fail))
2335 .expect("a FAIL update must be recorded");
2336 let error = fail
2337 .error()
2338 .expect("the FAIL update carries an error object");
2339 assert!(
2340 error.stack_trace().is_empty(),
2341 "the SDK captures no stack trace: {error:?}"
2342 );
2343 crate::error::WireError::new(error.error_type(), error.error_message())
2344 .with_error_data(error.error_data())
2345 }
2346
2347 fn payment_step(ctx: DurableContext) -> impl Future<Output = Result<i32, OperationError>> {
2350 let op_id = ctx.mint_id();
2351 StepExecution {
2352 ctx,
2353 op_id,
2354 name: Some("pay".to_owned()),
2355 retry_strategy: Some(Box::new(|_err: &StepError, attempt: u32| {
2356 if attempt >= 2 {
2357 RetryDecision::Stop
2358 } else {
2359 RetryDecision::retry(Duration::from_secs(1))
2360 }
2361 })),
2362 serdes: crate::serdes::JsonSerdes,
2363 semantics: StepSemantics::default(),
2364 sub_type: OperationSubType::new("Payment").expect("a valid subtype"),
2365 plugin_name: None,
2366 closure: |step_ctx: StepContext| async move {
2367 if step_ctx.attempt() == 1 {
2368 Err::<i32, BoxError>("transient".into())
2369 } else {
2370 Ok(7)
2371 }
2372 },
2373 _marker: std::marker::PhantomData,
2374 }
2375 .execute()
2376 }
2377
2378 #[tokio::test]
2383 async fn custom_sub_type_on_every_step_update() {
2384 let (ctx, first) = context_over(Vec::new());
2387 let signal = Arc::clone(ctx.coordinator());
2388 let outcome = crate::driver::test_support::outcome_of(
2389 ctx.clone(),
2390 Arc::clone(ctx.task_ownership()),
2391 signal,
2392 payment_step(ctx),
2393 )
2394 .await;
2395 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2396
2397 let (id, mut record) = step_record(CheckpointStatus::Ready, 1, None);
2399 record.sub_type = Some("Payment".to_owned());
2400 let (ctx, second) = context_over(vec![(id, record)]);
2401 assert_eq!(payment_step(ctx).await.expect("attempt 2 succeeds"), 7);
2402
2403 let updates: Vec<OperationUpdate> = first
2404 .recorded_updates()
2405 .into_iter()
2406 .chain(second.recorded_updates())
2407 .collect();
2408 let actions: Vec<OperationAction> = updates.iter().map(|u| u.action().clone()).collect();
2409 assert_eq!(
2410 actions,
2411 [
2412 OperationAction::Start,
2413 OperationAction::Retry,
2414 OperationAction::Start,
2415 OperationAction::Succeed
2416 ]
2417 );
2418 for update in &updates {
2419 assert_eq!(update.sub_type(), Some("Payment"), "{update:?}");
2420 }
2421 }
2422
2423 #[tokio::test]
2427 async fn custom_sub_type_on_rejected_write_and_no_terminal_fail() {
2428 use crate::client::TestResponse;
2429
2430 let (ctx, client) = context_over(Vec::new());
2431 client.enqueue_checkpoint_response(TestResponse::Success(vec![]));
2432 client.enqueue_checkpoint_response(TestResponse::NonRetryableError(
2433 "payload rejected".to_owned(),
2434 ));
2435 let op_id = ctx.mint_id();
2436 let exec = StepExecution {
2437 ctx: ctx.clone(),
2438 op_id,
2439 name: Some("pay".to_owned()),
2440 retry_strategy: None,
2441 serdes: crate::serdes::JsonSerdes,
2442 semantics: StepSemantics::default(),
2443 sub_type: OperationSubType::new("Payment").expect("a valid subtype"),
2444 plugin_name: None,
2445 closure: |_: StepContext| async { Ok::<i32, BoxError>(7) },
2446 _marker: std::marker::PhantomData,
2447 };
2448 let signal = Arc::clone(ctx.coordinator());
2449 let outcome = crate::driver::test_support::outcome_of(
2450 ctx.clone(),
2451 Arc::clone(ctx.task_ownership()),
2452 signal,
2453 exec.execute(),
2454 )
2455 .await;
2456 assert!(
2457 matches!(outcome, crate::driver::InvocationOutcome::Failed { .. }),
2458 "a rejected SUCCEED fails the execution: {outcome:?}"
2459 );
2460
2461 let updates = client.recorded_updates();
2462 let actions: Vec<OperationAction> = updates.iter().map(|u| u.action().clone()).collect();
2463 assert_eq!(actions, [OperationAction::Start, OperationAction::Succeed]);
2464 let rejected = updates.last().expect("the rejected SUCCEED is recorded");
2465 assert_eq!(rejected.sub_type(), Some("Payment"));
2466 }
2467
2468 async fn typed_failing_body(_: StepContext) -> Result<i32, BoxError> {
2470 Err(Box::new(
2471 crate::error::TypedError::with_type("PaymentError", "declined")
2472 .with_error_data(r#"{"code":42}"#),
2473 ))
2474 }
2475
2476 #[tokio::test]
2477 async fn step_failure_records_typed_error_data() {
2478 let (ctx, client) = context_over(Vec::new());
2480 let op_id = ctx.mint_id();
2481 let exec = StepExecution {
2482 ctx,
2483 op_id,
2484 name: None,
2485 retry_strategy: Some(Box::new(|_err: &StepError, _attempt: u32| {
2486 RetryDecision::Stop
2487 })),
2488 serdes: crate::serdes::JsonSerdes,
2489 semantics: StepSemantics::default(),
2490 sub_type: OperationSubType::STEP,
2491 plugin_name: None,
2492 closure: typed_failing_body,
2493 _marker: std::marker::PhantomData,
2494 };
2495 let live = exec.execute().await.expect_err("the step must fail");
2496 let recorded = recorded_fail(&client);
2497 assert_eq!(recorded.error_type(), Some("PaymentError"));
2498 assert_eq!(recorded.error_message(), Some("declined"));
2499 assert_eq!(recorded.error_data(), Some(r#"{"code":42}"#));
2500 assert_eq!(
2501 live.wire().and_then(crate::error::WireError::error_data),
2502 Some(r#"{"code":42}"#)
2503 );
2504 assert_eq!(
2505 live.wire().and_then(crate::error::WireError::error_message),
2506 Some("declined")
2507 );
2508
2509 let replayed_wire = crate::error::WireError::new(Some("PaymentError"), Some("declined"))
2511 .with_error_data(Some(r#"{"code":42}"#));
2512 let (ctx, _client) = context_over(vec![step_record(
2513 CheckpointStatus::Failed,
2514 0,
2515 Some(&replayed_wire),
2516 )]);
2517 let op_id = ctx.mint_id();
2518 let exec = StepExecution {
2519 ctx,
2520 op_id,
2521 name: None,
2522 retry_strategy: None,
2523 serdes: crate::serdes::JsonSerdes,
2524 semantics: StepSemantics::default(),
2525 sub_type: OperationSubType::STEP,
2526 plugin_name: None,
2527 closure: typed_failing_body,
2528 _marker: std::marker::PhantomData,
2529 };
2530 let err = exec
2531 .execute()
2532 .await
2533 .expect_err("the replayed step must fail");
2534 assert_eq!(
2535 err.wire().and_then(crate::error::WireError::error_data),
2536 Some(r#"{"code":42}"#)
2537 );
2538 assert_eq!(
2539 err.wire().and_then(crate::error::WireError::error_message),
2540 Some("declined")
2541 );
2542 }
2543
2544 #[tokio::test]
2545 async fn step_stop_returns_execution_failed_live_and_replay() {
2546 fn retry_once() -> RetryStrategy {
2547 Box::new(|_err: &StepError, attempt: u32| {
2548 if attempt >= 2 {
2549 RetryDecision::Stop
2550 } else {
2551 RetryDecision::retry(Duration::from_secs(1))
2552 }
2553 })
2554 }
2555 fn execution(ctx: DurableContext) -> impl Future<Output = Result<i32, OperationError>> {
2556 let op_id = ctx.mint_id();
2557 StepExecution {
2558 ctx,
2559 op_id,
2560 name: None,
2561 retry_strategy: Some(retry_once()),
2562 serdes: crate::serdes::JsonSerdes,
2563 semantics: StepSemantics::default(),
2564 sub_type: OperationSubType::STEP,
2565 plugin_name: None,
2566 closure: |_: StepContext| async { Err::<i32, BoxError>("boom".into()) },
2567 _marker: std::marker::PhantomData,
2568 }
2569 .execute()
2570 }
2571 fn assert_execution_failed(err: &OperationError) {
2572 let OperationErrorKind::Step(step_err) = err.kind() else {
2573 unreachable!("a step builds a step error: {err:?}");
2574 };
2575 assert!(
2576 matches!(step_err.kind(), StepErrorKind::ExecutionFailed),
2577 "unexpected kind: {:?}",
2578 step_err.kind()
2579 );
2580 let rendered = format!("{err:#}");
2581 assert!(rendered.ends_with("boom"), "got: {rendered}");
2582 assert!(!rendered.contains("retries exhausted"), "got: {rendered}");
2583 }
2584
2585 let (ctx, client) = context_over(Vec::new());
2587 let signal = Arc::clone(ctx.coordinator());
2588 let outcome = crate::driver::test_support::outcome_of(
2589 ctx.clone(),
2590 Arc::clone(ctx.task_ownership()),
2591 signal,
2592 execution(ctx),
2593 )
2594 .await;
2595 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2596 assert!(
2597 client
2598 .recorded_updates()
2599 .iter()
2600 .any(|u| matches!(u.action(), OperationAction::Retry)),
2601 "attempt 1 records a RETRY"
2602 );
2603
2604 let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Ready, 1, None)]);
2606 let live = execution(ctx).await.expect_err("the step must fail");
2607 assert_execution_failed(&live);
2608 assert_eq!(
2609 format!("{live:#}"),
2610 "operation error: step: execution failed: boom"
2611 );
2612
2613 let recorded = recorded_fail(&client);
2615 let (ctx, _client) = context_over(vec![step_record(
2616 CheckpointStatus::Failed,
2617 1,
2618 Some(&recorded),
2619 )]);
2620 let replayed = execution(ctx)
2621 .await
2622 .expect_err("the replayed step must fail");
2623 assert_execution_failed(&replayed);
2624 }
2625
2626 #[tokio::test]
2630 async fn step_live_failure_source_downcasts_to_concrete_user_type() {
2631 use crate::client::InMemoryExecutionClient;
2632
2633 #[derive(Debug)]
2634 struct PaymentDeclined {
2635 code: u16,
2636 }
2637 impl std::fmt::Display for PaymentDeclined {
2638 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2639 write!(f, "payment declined (code {})", self.code)
2640 }
2641 }
2642 impl std::error::Error for PaymentDeclined {}
2643
2644 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2645 let log = Arc::new(CheckpointLog::empty());
2646 let ctx = DurableContext::new_root_with_client(
2647 "arn:test".to_owned(),
2648 lambda_runtime::Context::default(),
2649 log,
2650 client,
2651 "token0".to_owned(),
2652 );
2653 let op_id = ctx.mint_id();
2654
2655 let no_retry: RetryStrategy =
2656 Box::new(|_err: &StepError, _attempt: u32| RetryDecision::Stop);
2657 let closure = |_: StepContext| async {
2658 Err::<i32, BoxError>(Box::new(PaymentDeclined { code: 402 }))
2659 };
2660
2661 let exec = StepExecution {
2662 ctx,
2663 op_id,
2664 name: None,
2665 retry_strategy: Some(no_retry),
2666 serdes: crate::serdes::JsonSerdes,
2667 semantics: StepSemantics::default(),
2668 sub_type: OperationSubType::STEP,
2669 plugin_name: None,
2670 closure,
2671 _marker: std::marker::PhantomData,
2672 };
2673
2674 let err = exec.execute().await.expect_err("step must fail");
2675
2676 let mut source: Option<&(dyn std::error::Error + 'static)> =
2678 std::error::Error::source(&err);
2679 let mut found = None;
2680 while let Some(e) = source {
2681 if let Some(declined) = e.downcast_ref::<PaymentDeclined>() {
2682 found = Some(declined);
2683 break;
2684 }
2685 source = e.source();
2686 }
2687 let declined = found.expect("caller's concrete error type must be reachable via source()");
2688 assert_eq!(declined.code, 402);
2689
2690 assert_eq!(err.status(), Some("FAILED"));
2692 let wire = err.wire().expect("live failure carries its wire record");
2693 assert_eq!(wire.error_type(), Some("Error"));
2694 assert!(
2695 wire.error_message()
2696 .is_some_and(|m| m.contains("payment declined (code 402)")),
2697 "wire message flattens the chain: {wire:?}"
2698 );
2699 assert!(
2700 wire.stack_trace().is_empty(),
2701 "the SDK captures no stack trace for a live failure"
2702 );
2703 }
2704
2705 #[tokio::test]
2706 async fn step_retry_schedules_and_suspends() {
2707 use crate::client::InMemoryExecutionClient;
2708
2709 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2710 let log = Arc::new(CheckpointLog::empty());
2711 let ctx = DurableContext::new_root_with_client(
2712 "arn:test".to_owned(),
2713 lambda_runtime::Context::default(),
2714 log,
2715 client,
2716 "token0".to_owned(),
2717 );
2718 let op_id = ctx.mint_id();
2719
2720 let always_retry: RetryStrategy = Box::new(|_err: &StepError, _attempt: u32| {
2722 RetryDecision::retry(Duration::from_secs(1))
2723 });
2724
2725 let closure = |_: StepContext| async { Err::<i32, BoxError>("transient".into()) };
2726
2727 let exec = StepExecution {
2728 ctx: ctx.clone(),
2729 op_id,
2730 name: None,
2731 retry_strategy: Some(always_retry),
2732 serdes: crate::serdes::JsonSerdes,
2733 semantics: StepSemantics::default(),
2734 sub_type: OperationSubType::STEP,
2735 plugin_name: None,
2736 closure,
2737 _marker: std::marker::PhantomData,
2738 };
2739
2740 let signal = Arc::clone(ctx.coordinator());
2741 let outcome = crate::driver::test_support::outcome_of(
2744 ctx.clone(),
2745 Arc::clone(ctx.task_ownership()),
2746 signal,
2747 exec.execute(),
2748 )
2749 .await;
2750 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2751 }
2752
2753 #[tokio::test]
2754 async fn step_retry_fractional_delay_rounds_up() {
2755 use crate::client::InMemoryExecutionClient;
2756
2757 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2758 let log = Arc::new(CheckpointLog::empty());
2759 let ctx = DurableContext::new_root_with_client(
2760 "arn:test".to_owned(),
2761 lambda_runtime::Context::default(),
2762 log,
2763 client.clone(),
2764 "token0".to_owned(),
2765 );
2766 let op_id = ctx.mint_id();
2767
2768 let fractional_retry: RetryStrategy = Box::new(|_err: &StepError, _attempt: u32| {
2772 RetryDecision::retry(Duration::from_millis(1900))
2773 });
2774
2775 let closure = |_: StepContext| async { Err::<i32, BoxError>("transient".into()) };
2776
2777 let exec = StepExecution {
2778 ctx: ctx.clone(),
2779 op_id,
2780 name: None,
2781 retry_strategy: Some(fractional_retry),
2782 serdes: crate::serdes::JsonSerdes,
2783 semantics: StepSemantics::default(),
2784 sub_type: OperationSubType::STEP,
2785 plugin_name: None,
2786 closure,
2787 _marker: std::marker::PhantomData,
2788 };
2789
2790 let signal = Arc::clone(ctx.coordinator());
2791 let outcome = crate::driver::test_support::outcome_of(
2792 ctx.clone(),
2793 Arc::clone(ctx.task_ownership()),
2794 signal,
2795 exec.execute(),
2796 )
2797 .await;
2798 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2799
2800 let updates = client.recorded_updates();
2802 let retry = updates
2803 .iter()
2804 .find(|u| matches!(u.action(), OperationAction::Retry))
2805 .expect("a Retry update must be recorded");
2806 let delay = retry
2807 .step_options()
2808 .and_then(aws_sdk_lambda::types::StepOptions::next_attempt_delay_seconds)
2809 .expect("Retry update must carry NextAttemptDelaySeconds");
2810 assert_eq!(delay, 2, "1.9s must round UP to 2s, not truncate to 1s");
2811 }
2812
2813 #[tokio::test]
2814 async fn step_spawn_executes_on_blessed_task() {
2815 use crate::client::InMemoryExecutionClient;
2816
2817 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2818 let log = Arc::new(CheckpointLog::empty());
2819 let ctx = DurableContext::new_root_with_client(
2820 "arn:test".to_owned(),
2821 lambda_runtime::Context::default(),
2822 log,
2823 client,
2824 "token0".to_owned(),
2825 );
2826
2827 let handle = ctx.step(|_| async { Ok(7i32) }).name("spawned").spawn();
2829 let result = handle.await;
2830 assert_eq!(result.unwrap(), 7);
2831 }
2832
2833 #[tokio::test]
2834 async fn step_ownership_rejection_from_foreign_task() {
2835 use crate::client::InMemoryExecutionClient;
2836
2837 let result = tokio::spawn(async {
2840 let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2841 let log = Arc::new(CheckpointLog::empty());
2842 let ctx = DurableContext::new_root_with_client(
2843 "arn:test".to_owned(),
2844 lambda_runtime::Context::default(),
2845 log,
2846 client,
2847 "token0".to_owned(),
2848 );
2849
2850 let ctx_clone = ctx.clone();
2852 let handle = tokio::spawn(async move {
2853 let op_id = ctx_clone.mint_id();
2854 let closure = |_: StepContext| async { Ok::<i32, BoxError>(1) };
2855
2856 let exec = StepExecution {
2857 ctx: ctx_clone,
2858 op_id,
2859 name: None,
2860 retry_strategy: None,
2861 serdes: crate::serdes::JsonSerdes,
2862 semantics: StepSemantics::default(),
2863 sub_type: OperationSubType::STEP,
2864 plugin_name: None,
2865 closure,
2866 _marker: std::marker::PhantomData,
2867 };
2868 exec.execute().await
2869 });
2870
2871 handle.await.unwrap()
2872 })
2873 .await;
2874
2875 let inner_result = result.unwrap();
2876 assert!(inner_result.is_err());
2877 let err_msg = format!("{:#}", inner_result.unwrap_err());
2878 assert!(
2879 err_msg.contains("task") || err_msg.contains("ownership"),
2880 "expected ownership error, got: {err_msg}"
2881 );
2882 }
2883
2884 fn kind_recording_strategy(
2889 decision: RetryDecision,
2890 ) -> (RetryStrategy, Arc<std::sync::Mutex<Vec<String>>>) {
2891 let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2892 let seen_clone = Arc::clone(&seen);
2893 let strategy: RetryStrategy = Box::new(move |err: &StepError, _attempt: u32| {
2894 seen_clone
2895 .lock()
2896 .expect("kind log lock")
2897 .push(err.kind().to_string());
2898 decision.clone()
2899 });
2900 (strategy, seen)
2901 }
2902
2903 fn recorded_error_for(
2906 client: &crate::client::InMemoryExecutionClient,
2907 action: &OperationAction,
2908 ) -> crate::error::WireError {
2909 let updates = client.recorded_updates();
2910 let update = updates
2911 .iter()
2912 .find(|u| u.action() == action)
2913 .unwrap_or_else(|| panic!("a {action:?} update must be recorded: {updates:?}"));
2914 let error = update.error().expect("the update carries an error object");
2915 crate::error::WireError::new(error.error_type(), error.error_message())
2916 }
2917
2918 fn at_most_once_over_started(
2921 ctx: DurableContext,
2922 retry_strategy: Option<RetryStrategy>,
2923 executed: Arc<std::sync::atomic::AtomicBool>,
2924 ) -> StepExecution<
2925 i32,
2926 impl FnOnce(StepContext) -> std::future::Ready<Result<i32, BoxError>> + Clone,
2927 crate::serdes::JsonSerdes,
2928 > {
2929 let op_id = ctx.mint_id();
2930 StepExecution {
2931 ctx,
2932 op_id,
2933 name: Some("at-most-once-step".to_owned()),
2934 retry_strategy,
2935 serdes: crate::serdes::JsonSerdes,
2936 semantics: StepSemantics::AtMostOncePerRetry,
2937 sub_type: OperationSubType::STEP,
2938 plugin_name: None,
2939 closure: move |_: StepContext| {
2940 executed.store(true, std::sync::atomic::Ordering::SeqCst);
2941 std::future::ready(Ok::<i32, BoxError>(42))
2942 },
2943 _marker: std::marker::PhantomData,
2944 }
2945 }
2946
2947 #[tokio::test]
2948 async fn step_at_most_once_replay_started_no_retry_fails_permanently() {
2949 use std::sync::atomic::{AtomicBool, Ordering};
2950
2951 let executed = Arc::new(AtomicBool::new(false));
2952 let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
2953 let (strategy, seen) = kind_recording_strategy(RetryDecision::Stop);
2954 let exec = at_most_once_over_started(ctx, Some(strategy), Arc::clone(&executed));
2955
2956 let err = exec
2957 .execute()
2958 .await
2959 .expect_err("expected failure for interrupted + no retry");
2960
2961 assert_eq!(*seen.lock().expect("kind log lock"), ["step interrupted"]);
2962 let fail = recorded_error_for(&client, &OperationAction::Fail);
2963 assert_eq!(fail.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
2964 assert_eq!(fail.error_message(), Some(STEP_INTERRUPTED_MESSAGE));
2965 let OperationErrorKind::Step(step_err) = err.kind() else {
2966 panic!("expected a step error, got {err:?}");
2967 };
2968 assert!(
2969 matches!(step_err.kind(), StepErrorKind::Interrupted),
2970 "got {step_err:?}"
2971 );
2972 assert_eq!(
2973 err.wire().and_then(crate::error::WireError::error_type),
2974 Some(STEP_INTERRUPTED_ERROR_TYPE)
2975 );
2976 assert!(
2977 !executed.load(Ordering::SeqCst),
2978 "closure should NOT execute when AtMostOncePerRetry and replay=Started"
2979 );
2980 }
2981
2982 #[tokio::test]
2983 async fn interrupted_attempt_gives_strategy_step_error_interrupted() {
2984 let executed = Arc::new(std::sync::atomic::AtomicBool::new(false));
2985 let (ctx, _client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
2986 let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2987 let seen_clone = Arc::clone(&seen);
2988 let strategy: RetryStrategy = Box::new(move |err: &StepError, _attempt: u32| {
2989 let source = std::error::Error::source(err).map(ToString::to_string);
2990 seen_clone
2991 .lock()
2992 .expect("strategy log lock")
2993 .push((err.kind().to_string(), source));
2994 RetryDecision::Stop
2995 });
2996 let exec = at_most_once_over_started(ctx, Some(strategy), executed);
2997
2998 exec.execute()
2999 .await
3000 .expect_err("an interrupted attempt with no retry fails");
3001
3002 let expected = StepError::interrupted();
3003 assert_eq!(
3004 *seen.lock().expect("strategy log lock"),
3005 [(
3006 expected.kind().to_string(),
3007 std::error::Error::source(&expected).map(ToString::to_string),
3008 )]
3009 );
3010 }
3011
3012 #[tokio::test]
3013 async fn step_at_most_once_replay_started_with_retry_suspends() {
3014 use std::sync::atomic::{AtomicBool, Ordering};
3015
3016 let executed = Arc::new(AtomicBool::new(false));
3017 let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
3018 let (strategy, seen) =
3019 kind_recording_strategy(RetryDecision::retry(Duration::from_secs(5)));
3020 let signal = Arc::clone(ctx.coordinator());
3021 let ownership = Arc::clone(ctx.task_ownership());
3022 let root = ctx.clone();
3023 let exec = at_most_once_over_started(ctx, Some(strategy), Arc::clone(&executed));
3024
3025 let outcome =
3028 crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
3029 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
3030 assert_eq!(*seen.lock().expect("kind log lock"), ["step interrupted"]);
3031 let retry = recorded_error_for(&client, &OperationAction::Retry);
3032 assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
3033 assert_eq!(retry.error_message(), Some(STEP_INTERRUPTED_MESSAGE));
3034 assert!(
3035 !executed.load(Ordering::SeqCst),
3036 "closure should NOT execute when AtMostOncePerRetry and replay=Started"
3037 );
3038 }
3039
3040 #[tokio::test]
3041 async fn step_at_most_once_interrupted_default_strategy_retries() {
3042 use std::sync::atomic::AtomicBool;
3043
3044 let executed = Arc::new(AtomicBool::new(false));
3045 let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
3046 let signal = Arc::clone(ctx.coordinator());
3047 let ownership = Arc::clone(ctx.task_ownership());
3048 let root = ctx.clone();
3049 let exec = at_most_once_over_started(ctx, None, executed);
3050
3051 let outcome =
3052 crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
3053 assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
3054 let retry = recorded_error_for(&client, &OperationAction::Retry);
3055 assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
3056 }
3057
3058 #[tokio::test]
3059 async fn step_body_error_strategy_sees_execution_failed() {
3060 let (ctx, client) = context_over(Vec::new());
3061 let op_id = ctx.mint_id();
3062 let (strategy, seen) = kind_recording_strategy(RetryDecision::Stop);
3063 let exec = StepExecution {
3064 ctx,
3065 op_id,
3066 name: None,
3067 retry_strategy: Some(strategy),
3068 serdes: crate::serdes::JsonSerdes,
3069 semantics: StepSemantics::AtMostOncePerRetry,
3070 sub_type: OperationSubType::STEP,
3071 plugin_name: None,
3072 closure: |_: StepContext| async { Err::<i32, BoxError>("boom".into()) },
3073 _marker: std::marker::PhantomData,
3074 };
3075
3076 let err = exec.execute().await.expect_err("the step must fail");
3077
3078 assert_eq!(*seen.lock().expect("kind log lock"), ["execution failed"]);
3079 let fail = recorded_error_for(&client, &OperationAction::Fail);
3080 assert_eq!(fail.error_type(), Some(STEP_FALLBACK_ERROR_TYPE));
3081 let OperationErrorKind::Step(step_err) = err.kind() else {
3082 panic!("expected a step error, got {err:?}");
3083 };
3084 assert!(
3085 matches!(step_err.kind(), StepErrorKind::ExecutionFailed),
3086 "got {step_err:?}"
3087 );
3088 }
3089
3090 #[tokio::test]
3091 async fn step_interrupted_fail_record_replays_as_interrupted() {
3092 let wire = crate::error::WireError::new(
3093 Some(STEP_INTERRUPTED_ERROR_TYPE),
3094 Some(STEP_INTERRUPTED_MESSAGE),
3095 );
3096 let (ctx, _client) =
3097 context_over(vec![step_record(CheckpointStatus::Failed, 1, Some(&wire))]);
3098 let op_id = ctx.mint_id();
3099 let exec = StepExecution {
3100 ctx,
3101 op_id,
3102 name: None,
3103 retry_strategy: None,
3104 serdes: crate::serdes::JsonSerdes,
3105 semantics: StepSemantics::AtMostOncePerRetry,
3106 sub_type: OperationSubType::STEP,
3107 plugin_name: None,
3108 closure: |_: StepContext| async { Ok::<i32, BoxError>(1) },
3109 _marker: std::marker::PhantomData,
3110 };
3111
3112 let err = exec
3113 .execute()
3114 .await
3115 .expect_err("the replayed step must fail");
3116
3117 let OperationErrorKind::Step(step_err) = err.kind() else {
3118 panic!("expected a step error, got {err:?}");
3119 };
3120 assert!(
3121 matches!(step_err.kind(), StepErrorKind::Interrupted),
3122 "got {step_err:?}"
3123 );
3124 assert_eq!(
3125 err.wire().and_then(crate::error::WireError::error_type),
3126 Some(STEP_INTERRUPTED_ERROR_TYPE)
3127 );
3128 let replayed = std::error::Error::source(step_err)
3129 .and_then(|s| s.downcast_ref::<crate::error::ReplayedFailure>())
3130 .expect("replay source is a ReplayedFailure");
3131 assert_eq!(replayed.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
3132 }
3133
3134 #[tokio::test]
3135 async fn step_at_least_once_replay_started_re_executes() {
3136 use crate::engine::{CheckpointLog, CheckpointRecord};
3138 use std::sync::atomic::{AtomicBool, Ordering};
3139
3140 let executed = Arc::new(AtomicBool::new(false));
3141 let executed_clone = Arc::clone(&executed);
3142
3143 let wire_key = crate::engine::compute_wire_id_public("1");
3144 let record = CheckpointRecord {
3145 id: wire_key.clone(),
3146 status: CheckpointStatus::Started,
3147 result: None,
3148 error_type: None,
3149 error_message: None,
3150 error_data: None,
3151 stack_trace: None,
3152 attempt: 0,
3153 invoke_result: None,
3154 invoke_error_type: None,
3155 invoke_error_message: None,
3156 invoke_error_data: None,
3157 invoke_stack_trace: None,
3158 replay_children: false,
3159 callback_id: None,
3160 op_type: None,
3161 sub_type: None,
3162 due_at: None,
3163 op_name: None,
3164 parent_id: None,
3165 start_timestamp: None,
3166 end_timestamp: None,
3167 };
3168 let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
3169
3170 let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
3171 let ctx = DurableContext::new_root_with_client(
3172 "arn:test".to_owned(),
3173 lambda_runtime::Context::default(),
3174 log,
3175 client,
3176 "token0".to_owned(),
3177 );
3178 let op_id = ctx.mint_id();
3179
3180 let closure = move |_: StepContext| {
3181 executed_clone.store(true, Ordering::SeqCst);
3182 async { Ok::<i32, BoxError>(77) }
3183 };
3184
3185 let exec = StepExecution {
3186 ctx,
3187 op_id,
3188 name: None,
3189 retry_strategy: None,
3190 serdes: crate::serdes::JsonSerdes,
3191 semantics: StepSemantics::AtLeastOncePerRetry,
3192 sub_type: OperationSubType::STEP,
3193 plugin_name: None,
3194 closure,
3195 _marker: std::marker::PhantomData,
3196 };
3197
3198 let result = exec.execute().await;
3199 assert_eq!(result.unwrap(), 77);
3200 assert!(
3201 executed.load(Ordering::SeqCst),
3202 "closure SHOULD execute under AtLeastOncePerRetry with Started replay"
3203 );
3204 }
3205
3206 use crate::builders::{Backoff, JitterStrategy, RetryStrategyConfig};
3209
3210 fn sample_error() -> StepError {
3211 StepError::new(StepErrorKind::ExecutionFailed, Some("boom".into()))
3212 }
3213
3214 fn retry_secs(decision: &RetryDecision) -> u64 {
3216 match decision {
3217 RetryDecision::Retry { delay } => delay.as_secs(),
3218 RetryDecision::Stop => panic!("expected Retry, got Stop"),
3219 }
3220 }
3221
3222 #[test]
3227 fn default_config_without_jitter_matches_documented_constants() {
3228 let config = RetryStrategyConfig::builder()
3229 .max_attempts(6)
3230 .max_delay(Duration::from_mins(1))
3231 .jitter(JitterStrategy::None)
3232 .build();
3233
3234 let expected = [5u64, 10, 20, 40, 60];
3236 for (attempt, expected_secs) in (1u32..).zip(expected) {
3237 assert_eq!(
3238 retry_secs(&config.decide(&sample_error(), attempt)),
3239 expected_secs,
3240 "attempt {attempt}"
3241 );
3242 }
3243 assert_eq!(config.decide(&sample_error(), 6), RetryDecision::Stop);
3244 assert_eq!(config.decide(&sample_error(), 7), RetryDecision::Stop);
3245 }
3246
3247 #[test]
3252 fn default_retry_strategy_matches_default_config_modulo_jitter() {
3253 let strategy = default_retry_strategy();
3254 let no_jitter = RetryStrategyConfig::builder()
3255 .max_attempts(6)
3256 .max_delay(Duration::from_mins(1))
3257 .jitter(JitterStrategy::None)
3258 .build();
3259 let err = sample_error();
3260
3261 for attempt in 1u32..=5 {
3262 let base = retry_secs(&no_jitter.decide(&sample_error(), attempt));
3264 for _ in 0..50 {
3265 let secs = retry_secs(&strategy(&err, attempt));
3266 assert!(
3267 (1..=base).contains(&secs),
3268 "attempt {attempt}: delay {secs}s outside full-jitter envelope [1, {base}]"
3269 );
3270 }
3271 }
3272 assert_eq!(strategy(&err, 6), RetryDecision::Stop);
3274 assert_eq!(no_jitter.decide(&sample_error(), 6), RetryDecision::Stop);
3275 assert_eq!(strategy(&err, 7), RetryDecision::Stop);
3276 }
3277
3278 #[test]
3280 fn half_jitter_delays_stay_within_bounds() {
3281 let config = RetryStrategyConfig::builder()
3282 .max_attempts(6)
3283 .initial_delay(Duration::from_secs(8))
3284 .max_delay(Duration::from_secs(64))
3285 .backoff_rate(2.0)
3286 .jitter(JitterStrategy::Half)
3287 .build();
3288
3289 for attempt in 1u32..=3 {
3290 let base = 8u64 << (attempt - 1);
3292 for _ in 0..100 {
3293 let secs = retry_secs(&config.decide(&sample_error(), attempt));
3294 assert!(
3295 (base / 2..=base).contains(&secs),
3296 "attempt {attempt}: delay {secs}s outside half-jitter bounds \
3297 [{}, {base}]",
3298 base / 2
3299 );
3300 }
3301 }
3302 }
3303
3304 #[test]
3307 fn delays_are_capped_and_have_one_second_floor() {
3308 let tiny = RetryStrategyConfig::builder()
3310 .initial_delay(Duration::from_millis(1))
3311 .jitter(JitterStrategy::None)
3312 .build();
3313 assert_eq!(retry_secs(&tiny.decide(&sample_error(), 1)), 1);
3314
3315 let capped = RetryStrategyConfig::builder()
3317 .initial_delay(Duration::from_secs(30))
3318 .max_delay(Duration::from_secs(45))
3319 .backoff_rate(100.0)
3320 .jitter(JitterStrategy::None)
3321 .build();
3322 assert_eq!(retry_secs(&capped.decide(&sample_error(), 1)), 30);
3323 assert_eq!(retry_secs(&capped.decide(&sample_error(), 2)), 45);
3324 }
3325
3326 #[test]
3329 fn max_attempts_one_never_retries() {
3330 let config = RetryStrategyConfig::builder().max_attempts(1).build();
3331 assert_eq!(config.decide(&sample_error(), 1), RetryDecision::Stop);
3332 }
3333
3334 #[test]
3338 fn quantize_delay_rounds_to_nearest_with_one_second_floor() {
3339 let cases = [
3340 (0.0, 1u64),
3341 (0.4, 1),
3342 (1.1, 1),
3343 (2.25, 2),
3344 (2.5, 3),
3345 (4.4, 4),
3346 (5.0, 5),
3347 (f64::NAN, 1),
3348 (f64::INFINITY, 1),
3349 ];
3350 for (input, expected) in cases {
3351 assert_eq!(
3352 quantize_delay(input),
3353 Duration::from_secs(expected),
3354 "input {input}"
3355 );
3356 }
3357 }
3358
3359 #[test]
3363 fn fractional_delays_round_to_nearest() {
3364 let config = RetryStrategyConfig::builder()
3365 .initial_delay(Duration::from_millis(1100))
3366 .jitter(JitterStrategy::None)
3367 .build();
3368 assert_eq!(retry_secs(&config.decide(&sample_error(), 1)), 1);
3369 assert_eq!(retry_secs(&config.decide(&sample_error(), 2)), 2);
3371
3372 let slow = RetryStrategyConfig::builder()
3374 .max_attempts(6)
3375 .initial_delay(Duration::from_secs(1))
3376 .backoff_rate(1.5)
3377 .jitter(JitterStrategy::None)
3378 .build();
3379 assert_eq!(retry_secs(&slow.decide(&sample_error(), 3)), 2);
3380
3381 let half = RetryStrategyConfig::builder()
3382 .initial_delay(Duration::from_millis(2500))
3383 .jitter(JitterStrategy::None)
3384 .build();
3385 assert_eq!(retry_secs(&half.decide(&sample_error(), 1)), 3);
3386 }
3387
3388 #[test]
3393 fn linear_backoff_cases() {
3394 let linear = |initial: Duration, increment: Duration| {
3395 RetryStrategyConfig::builder()
3396 .initial_delay(initial)
3397 .backoff(Backoff::linear(increment))
3398 .jitter(JitterStrategy::None)
3399 };
3400 let delays = |config: &RetryStrategyConfig, attempts: std::ops::RangeInclusive<u32>| {
3401 attempts
3402 .map(|attempt| retry_secs(&config.decide(&sample_error(), attempt)))
3403 .collect::<Vec<_>>()
3404 };
3405
3406 let preset = RetryStrategyConfig::linear();
3408 assert_eq!(delays(&preset, 1..=5), [1, 2, 3, 4, 5], "case A");
3409 assert_eq!(
3411 preset.decide(&sample_error(), 6),
3412 RetryDecision::Stop,
3413 "case B"
3414 );
3415
3416 let capped = linear(Duration::from_secs(1), Duration::from_secs(10))
3418 .max_delay(Duration::from_secs(15))
3419 .max_attempts(6)
3420 .build();
3421 assert_eq!(delays(&capped, 1..=3), [1, 11, 15], "case C");
3422
3423 let fractional = linear(Duration::from_secs(1), Duration::from_millis(250))
3425 .max_attempts(6)
3426 .build();
3427 assert_eq!(delays(&fractional, 1..=4), [1, 1, 2, 2], "case D");
3428
3429 let zero = linear(Duration::ZERO, Duration::ZERO)
3431 .max_attempts(3)
3432 .build();
3433 assert_eq!(delays(&zero, 1..=1), [1], "case E");
3434
3435 assert_eq!(
3437 RetryStrategyConfig::no_retry().decide(&sample_error(), 1),
3438 RetryDecision::Stop,
3439 "case F"
3440 );
3441
3442 let exponential = RetryStrategyConfig::builder()
3444 .max_attempts(6)
3445 .max_delay(Duration::from_mins(1))
3446 .jitter(JitterStrategy::None)
3447 .build();
3448 assert_eq!(delays(&exponential, 1..=5), [5, 10, 20, 40, 60], "case G");
3449 }
3450
3451 #[test]
3454 fn linear_half_jitter_stays_within_bounds() {
3455 let config = RetryStrategyConfig::builder()
3456 .initial_delay(Duration::from_secs(8))
3457 .backoff(Backoff::linear(Duration::from_secs(8)))
3458 .jitter(JitterStrategy::Half)
3459 .build();
3460
3461 for (attempt, base) in [(1u32, 8u64), (2, 16), (3, 24)] {
3462 for _ in 0..100 {
3463 let secs = retry_secs(&config.decide(&sample_error(), attempt));
3464 assert!(
3465 (base / 2..=base).contains(&secs),
3466 "attempt {attempt}: delay {secs}s outside half-jitter bounds \
3467 [{}, {base}]",
3468 base / 2
3469 );
3470 }
3471 }
3472 }
3473
3474 #[derive(Debug)]
3476 struct Transient;
3477
3478 impl std::fmt::Display for Transient {
3479 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3480 f.write_str("transient")
3481 }
3482 }
3483
3484 impl std::error::Error for Transient {}
3485
3486 fn is_transient(err: &StepError) -> bool {
3487 std::error::Error::source(err)
3488 .and_then(|e| e.downcast_ref::<Transient>())
3489 .is_some()
3490 }
3491
3492 fn mentions_timeout(err: &StepError) -> bool {
3493 std::error::Error::source(err)
3494 .map(ToString::to_string)
3495 .is_some_and(|message| message.contains("timeout"))
3496 }
3497
3498 fn body_error(source: crate::error::Source) -> StepError {
3499 StepError::new(StepErrorKind::ExecutionFailed, Some(source))
3500 }
3501
3502 #[test]
3508 fn retry_if_filter_cases() {
3509 use std::sync::Arc;
3510 use std::sync::atomic::{AtomicU32, Ordering};
3511
3512 type Predicate = fn(&StepError) -> bool;
3513 type Case = (
3515 &'static str,
3516 Option<Predicate>,
3517 StepError,
3518 u32,
3519 Option<u64>,
3520 bool,
3521 );
3522 let either: Predicate = |err| is_transient(err) || mentions_timeout(err);
3523 let boom = || body_error("boom".into());
3524 let transient = || body_error(Box::new(Transient));
3525 let timeout = || body_error("read timeout".into());
3526
3527 let cases: Vec<Case> = vec![
3528 ("A", None, boom(), 1, Some(1), false),
3529 ("B", Some(|_| false), boom(), 1, None, true),
3530 ("C", Some(|_| true), boom(), 2, Some(2), true),
3531 ("D", Some(|_| true), boom(), 3, None, false),
3532 ("E", Some(is_transient), transient(), 1, Some(1), true),
3533 ("F", Some(is_transient), boom(), 1, None, true),
3534 ("G", Some(mentions_timeout), timeout(), 1, Some(1), true),
3535 ("H", Some(either), boom(), 1, None, true),
3536 ("I", Some(either), timeout(), 1, Some(1), true),
3537 ];
3538
3539 for (case, predicate, err, attempt, expected, called) in cases {
3540 let calls = Arc::new(AtomicU32::new(0));
3541 let mut builder = RetryStrategyConfig::builder()
3542 .max_attempts(3)
3543 .initial_delay(Duration::from_secs(1))
3544 .jitter(JitterStrategy::None);
3545 if let Some(predicate) = predicate {
3546 let calls = Arc::clone(&calls);
3547 builder = builder.retry_if(move |err| {
3548 calls.fetch_add(1, Ordering::SeqCst);
3549 predicate(err)
3550 });
3551 }
3552 let config = builder.build();
3553
3554 let decision = config.decide(&err, attempt);
3555 let expected = expected.map_or(RetryDecision::Stop, |secs| {
3556 RetryDecision::retry(Duration::from_secs(secs))
3557 });
3558 assert_eq!(decision, expected, "case {case}");
3559 assert_eq!(
3560 calls.load(Ordering::SeqCst),
3561 u32::from(called),
3562 "case {case}: predicate calls"
3563 );
3564 }
3565 }
3566
3567 #[test]
3570 fn retry_if_sees_the_error_kind() {
3571 let config = RetryStrategyConfig::builder()
3572 .max_attempts(3)
3573 .initial_delay(Duration::from_secs(1))
3574 .jitter(JitterStrategy::None)
3575 .retry_if(|err| matches!(err.kind(), StepErrorKind::ExecutionFailed))
3576 .build();
3577 let interrupted = StepError::new(StepErrorKind::Interrupted, None);
3578 assert_eq!(config.decide(&interrupted, 1), RetryDecision::Stop);
3579 assert_eq!(
3580 config.decide(&sample_error(), 1),
3581 RetryDecision::retry(Duration::from_secs(1))
3582 );
3583 }
3584
3585 #[test]
3587 fn into_retry_strategy_applies_retry_if() {
3588 let strategy = RetryStrategyConfig::builder()
3589 .retry_if(|_| false)
3590 .build()
3591 .into_retry_strategy();
3592 assert_eq!(strategy(&sample_error(), 1), RetryDecision::Stop);
3593 }
3594}
3595
3596#[cfg(test)]
3597mod coordinator_tests {
3598 use crate::driver::InvocationOutcome;
3599 use crate::driver::test_support::{
3600 RefreshClient, RefreshRequest, assert_near, drive_handler, named, operation_like,
3601 poll_once, refresh_ctx, wire_err,
3602 };
3603 use aws_sdk_lambda::types::{
3604 ErrorObject, Operation, OperationAction, OperationStatus, OperationType, OperationUpdate,
3605 StepDetails,
3606 };
3607 use std::sync::atomic::{AtomicUsize, Ordering};
3608 use std::sync::{Arc, Mutex};
3609 use std::time::Duration;
3610
3611 fn secs(n: u64) -> Duration {
3612 Duration::from_secs(n)
3613 }
3614
3615 fn step_record(
3618 req: &RefreshRequest<'_>,
3619 update: &OperationUpdate,
3620 status: OperationStatus,
3621 next_in: Option<Duration>,
3622 ) -> Operation {
3623 let mut details = StepDetails::builder().attempt(1);
3624 if let Some(next_in) = next_in {
3625 details = details.next_attempt_timestamp(req.timestamp_in(next_in));
3626 }
3627 operation_like(update, status)
3628 .step_details(details.build())
3629 .build()
3630 .expect("all required fields set")
3631 }
3632
3633 fn retry_client(name: &'static str) -> RefreshClient {
3636 RefreshClient::new(move |req| {
3637 let mut ops = Vec::new();
3638 if let Some(retry) = named(req.updates, name, &OperationAction::Retry) {
3639 ops.push(step_record(
3640 req,
3641 retry,
3642 OperationStatus::Pending,
3643 Some(secs(1)),
3644 ));
3645 }
3646 if req.empty_number == 1
3647 && let Some(retry) = named(req.history, name, &OperationAction::Retry)
3648 {
3649 ops.push(step_record(req, retry, OperationStatus::Ready, None));
3650 }
3651 Ok(ops)
3652 })
3653 }
3654
3655 fn retry_once(_err: &crate::StepError, attempt: u32) -> crate::RetryDecision {
3657 if attempt < 2 {
3658 crate::RetryDecision::retry(secs(1))
3659 } else {
3660 crate::RetryDecision::Stop
3661 }
3662 }
3663
3664 async fn sleeping_step(ctx: &crate::DurableContext) -> Result<u32, crate::error::WireError> {
3666 ctx.step(|_| async {
3667 tokio::time::sleep(secs(3)).await;
3668 Ok(1_u32)
3669 })
3670 .name("s")
3671 .await
3672 .map_err(wire_err)
3673 }
3674
3675 #[tokio::test(start_paused = true)]
3676 async fn step_retry_runs_next_attempt_in_invocation() {
3677 let client = Arc::new(retry_client("flaky"));
3678 let ctx = refresh_ctx(&client);
3679 let start = tokio::time::Instant::now();
3680 let attempts = Arc::new(Mutex::new(Vec::new()));
3681 let body_attempts = Arc::clone(&attempts);
3682 let outcome = drive_handler(ctx, None, move |ctx| async move {
3683 let mut flaky = std::pin::pin!(
3684 ctx.step(move |step_ctx| {
3685 let attempts = Arc::clone(&body_attempts);
3686 async move {
3687 attempts
3688 .lock()
3689 .expect("lock")
3690 .push((step_ctx.attempt(), start.elapsed()));
3691 if step_ctx.attempt() < 2 {
3692 return Err::<u32, crate::BoxError>("transient".into());
3693 }
3694 Ok(7)
3695 }
3696 })
3697 .name("flaky")
3698 .retry_strategy(retry_once)
3699 .spawn()
3700 );
3701 poll_once(flaky.as_mut()).await;
3702 sleeping_step(&ctx).await?;
3703 let value = flaky.await.map_err(wire_err)?;
3704 Ok(value.to_string())
3705 })
3706 .await
3707 .expect("no panic");
3708 assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3709 let attempts = attempts.lock().expect("lock").clone();
3710 let numbers: Vec<u32> = attempts.iter().map(|(n, _)| *n).collect();
3711 assert_eq!(numbers, vec![1, 2]);
3712 assert_near(attempts.get(1).expect("attempt 2 ran").1, secs(1));
3713 }
3714
3715 #[tokio::test(start_paused = true)]
3716 async fn step_retry_in_invocation_fires_start() {
3717 use crate::plugin::AttemptOutcome::{Failed, Succeeded};
3718 let client = Arc::new(retry_client("flaky"));
3719 let hooks = crate::plugin::test_support::OpHooks::default();
3720 let ctx = crate::DurableContext::new_root_with_client_and_defaults(
3721 "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
3722 lambda_runtime::Context::default(),
3723 Arc::new(crate::engine::CheckpointLog::empty()),
3724 Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
3725 "t0".to_owned(),
3726 Duration::ZERO,
3727 0,
3728 Some(hooks.plugins()),
3729 );
3730 let outcome = drive_handler(ctx, None, move |ctx| async move {
3731 let mut flaky = std::pin::pin!(
3732 ctx.step(|step_ctx| async move {
3733 if step_ctx.attempt() < 2 {
3734 return Err::<u32, crate::BoxError>("transient".into());
3735 }
3736 Ok(7)
3737 })
3738 .name("flaky")
3739 .retry_strategy(retry_once)
3740 .spawn()
3741 );
3742 poll_once(flaky.as_mut()).await;
3743 sleeping_step(&ctx).await?;
3744 let value = flaky.await.map_err(wire_err)?;
3745 Ok(value.to_string())
3746 })
3747 .await
3748 .expect("no panic");
3749 assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3750 let records = hooks.of("flaky");
3751 let shapes: Vec<_> = records
3752 .iter()
3753 .map(crate::plugin::test_support::OpRecord::shape)
3754 .collect();
3755 assert_eq!(
3756 shapes,
3757 [
3758 ("start", false, None, None),
3759 ("attempt_start", false, Some(1), None),
3760 ("attempt_end", false, Some(1), Some(Failed)),
3761 ("start", true, None, None),
3762 ("attempt_start", false, Some(2), None),
3763 ("attempt_end", false, Some(2), Some(Succeeded)),
3764 ("end", false, None, None),
3765 ]
3766 );
3767 let error = records.get(2).and_then(|r| r.error.as_deref());
3768 assert_eq!(error, Some("transient"));
3769 let result = records.get(5).and_then(|r| r.result.as_deref());
3772 assert_eq!(result, Some("7"));
3773 let statuses: Vec<_> = records.iter().map(|r| r.status).collect();
3774 assert_eq!(
3775 statuses,
3776 [
3777 Some(crate::OperationStatus::Started),
3778 Some(crate::OperationStatus::Started),
3779 Some(crate::OperationStatus::Pending),
3780 Some(crate::OperationStatus::Started),
3781 Some(crate::OperationStatus::Started),
3782 Some(crate::OperationStatus::Succeeded),
3783 Some(crate::OperationStatus::Succeeded),
3784 ]
3785 );
3786 }
3787
3788 #[tokio::test(start_paused = true)]
3793 async fn attempt_info_shared_by_hooks() {
3794 let client = Arc::new(retry_client("flaky"));
3795 let hooks = crate::plugin::recorder::UfHooks::default();
3796 let ctx = crate::DurableContext::new_root_with_client_and_defaults(
3797 "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
3798 lambda_runtime::Context::default(),
3799 Arc::new(crate::engine::CheckpointLog::empty()),
3800 Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
3801 "t0".to_owned(),
3802 Duration::ZERO,
3803 0,
3804 Some(hooks.plugins()),
3805 );
3806 let outcome = drive_handler(ctx, None, move |ctx| async move {
3807 let mut flaky = std::pin::pin!(
3808 ctx.step(|step_ctx| async move {
3809 if step_ctx.attempt() < 2 {
3810 return Err::<u32, crate::BoxError>("transient".into());
3811 }
3812 Ok(7)
3813 })
3814 .name("flaky")
3815 .retry_strategy(retry_once)
3816 .spawn()
3817 );
3818 poll_once(flaky.as_mut()).await;
3819 sleeping_step(&ctx).await?;
3820 let value = flaky.await.map_err(wire_err)?;
3821 Ok(value.to_string())
3822 })
3823 .await
3824 .expect("no panic");
3825 assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3826 let records = hooks.of("flaky");
3827 let starts: Vec<_> = records.iter().filter(|r| r.hook == "start").collect();
3828 assert_eq!(starts.len(), 2, "{records:?}");
3829 assert!(starts.get(1).is_some_and(|r| r.replay), "{starts:?}");
3830 let second = records
3831 .iter()
3832 .position(|r| r.hook == "attempt_start" && r.attempt == Some(2))
3833 .expect("attempt 2 started");
3834 let shared: Vec<_> = records
3835 .get(second..)
3836 .unwrap_or_default()
3837 .iter()
3838 .filter(|r| {
3839 matches!(
3840 r.hook,
3841 "attempt_start" | "uf_start" | "wrap_enter" | "wrap_exit" | "uf_end"
3842 )
3843 })
3844 .collect();
3845 let hook_names: Vec<_> = shared.iter().map(|r| r.hook).collect();
3846 assert_eq!(
3847 hook_names,
3848 [
3849 "attempt_start",
3850 "uf_start",
3851 "wrap_enter",
3852 "wrap_exit",
3853 "uf_end"
3854 ]
3855 );
3856 let first = shared.first().expect("attempt_start");
3857 for record in &shared {
3858 assert_eq!(
3859 record.info_addr, first.info_addr,
3860 "{} shares the info",
3861 record.hook
3862 );
3863 assert_eq!(record.status, Some(crate::OperationStatus::Started));
3864 assert_eq!(record.info_attempt, Some(2));
3865 assert!(!record.replay, "{}", record.hook);
3866 assert!(record.has_start_timestamp, "{}", record.hook);
3867 }
3868 }
3869
3870 #[tokio::test(start_paused = true)]
3871 async fn each_attempt_starts_from_original_captures() {
3872 let client = Arc::new(retry_client("vec"));
3873 let ctx = refresh_ctx(&client);
3874 let outcome = drive_handler(ctx, None, |ctx| async move {
3875 let v = vec![0_u32];
3876 let mut step = std::pin::pin!(
3877 ctx.step(move |step_ctx| async move {
3878 let mut v = v;
3879 v.push(step_ctx.attempt());
3880 if step_ctx.attempt() < 2 {
3881 return Err::<Vec<u32>, crate::BoxError>("transient".into());
3882 }
3883 Ok(v)
3884 })
3885 .name("vec")
3886 .retry_strategy(retry_once)
3887 .spawn()
3888 );
3889 poll_once(step.as_mut()).await;
3890 sleeping_step(&ctx).await?;
3891 let v = step.await.map_err(wire_err)?;
3892 Ok(format!("{v:?}"))
3893 })
3894 .await
3895 .expect("no panic");
3896 assert_eq!(outcome, InvocationOutcome::Complete("[0, 2]".to_owned()));
3897 }
3898
3899 #[tokio::test(start_paused = true)]
3900 async fn arc_capture_is_shared_across_attempts_in_invocation() {
3901 let client = Arc::new(retry_client("arc"));
3902 let ctx = refresh_ctx(&client);
3903 let counter = Arc::new(AtomicUsize::new(0));
3904 let handler_counter = Arc::clone(&counter);
3905 let outcome = drive_handler(ctx, None, move |ctx| async move {
3906 let mut step = std::pin::pin!(
3907 ctx.step(move |step_ctx| async move {
3908 handler_counter.fetch_add(1, Ordering::SeqCst);
3909 if step_ctx.attempt() < 2 {
3910 return Err::<u32, crate::BoxError>("transient".into());
3911 }
3912 Ok(1)
3913 })
3914 .name("arc")
3915 .retry_strategy(retry_once)
3916 .spawn()
3917 );
3918 poll_once(step.as_mut()).await;
3919 sleeping_step(&ctx).await?;
3920 step.await.map_err(wire_err)?;
3921 Ok("done".to_owned())
3922 })
3923 .await
3924 .expect("no panic");
3925 assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
3926 assert_eq!(counter.load(Ordering::SeqCst), 2);
3927 }
3928
3929 #[tokio::test(start_paused = true)]
3930 async fn lone_step_retry_suspends() {
3931 let client = Arc::new(retry_client("lone"));
3932 let ctx = refresh_ctx(&client);
3933 let runs = Arc::new(AtomicUsize::new(0));
3934 let body_runs = Arc::clone(&runs);
3935 let outcome = drive_handler(ctx, None, move |ctx| async move {
3936 ctx.step(move |_| async move {
3937 body_runs.fetch_add(1, Ordering::SeqCst);
3938 Err::<u32, crate::BoxError>("transient".into())
3939 })
3940 .name("lone")
3941 .retry_strategy(retry_once)
3942 .await
3943 .map_err(wire_err)?;
3944 Ok("done".to_owned())
3945 })
3946 .await
3947 .expect("no panic");
3948 assert_eq!(outcome, InvocationOutcome::Pending);
3949 assert_eq!(runs.load(Ordering::SeqCst), 1, "the body ran once");
3950 assert!(client.empty_calls().is_empty(), "no refresh was sent");
3951 }
3952
3953 #[tokio::test(start_paused = true)]
3954 async fn replayed_pending_step_runs_next_attempt_in_invocation() {
3955 let client = Arc::new(RefreshClient::new(|req| {
3956 let mut ops = Vec::new();
3957 if req.empty_number == 1 {
3958 ops.push(
3959 operation_like(&flaky_update(), OperationStatus::Ready)
3960 .step_details(StepDetails::builder().attempt(1).build())
3961 .build()
3962 .expect("all required fields set"),
3963 );
3964 }
3965 Ok(ops)
3966 }));
3967 let ctx = pending_flaky_ctx(&client);
3968 let attempts = Arc::new(Mutex::new(Vec::new()));
3969 let body_attempts = Arc::clone(&attempts);
3970 let outcome = drive_handler(ctx, None, move |ctx| async move {
3971 let mut flaky = std::pin::pin!(
3972 ctx.step(move |step_ctx| {
3973 let attempts = Arc::clone(&body_attempts);
3974 async move {
3975 attempts.lock().expect("lock").push(step_ctx.attempt());
3976 Ok::<u32, crate::BoxError>(7)
3977 }
3978 })
3979 .name("flaky")
3980 .retry_strategy(retry_once)
3981 .spawn()
3982 );
3983 poll_once(flaky.as_mut()).await;
3984 sleeping_step(&ctx).await?;
3985 let value = flaky.await.map_err(wire_err)?;
3986 Ok(value.to_string())
3987 })
3988 .await
3989 .expect("no panic");
3990 assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3991 assert_eq!(*attempts.lock().expect("lock"), vec![2]);
3992 }
3993
3994 fn pending_flaky_ctx(client: &Arc<RefreshClient>) -> crate::DurableContext {
3997 let pending = operation_like(&flaky_update(), OperationStatus::Pending)
3998 .step_details(
3999 StepDetails::builder()
4000 .attempt(1)
4001 .next_attempt_timestamp(aws_smithy_types::DateTime::from(
4002 std::time::SystemTime::now() + secs(1),
4003 ))
4004 .build(),
4005 )
4006 .build()
4007 .expect("all required fields set");
4008 crate::DurableContext::new_root_with_client(
4009 "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
4010 lambda_runtime::Context::default(),
4011 Arc::new(crate::client::operations_to_checkpoint_log(&[pending])),
4012 Arc::clone(client) as Arc<dyn crate::client::ExecutionClient>,
4013 "t0".to_owned(),
4014 )
4015 }
4016
4017 #[tokio::test(start_paused = true)]
4021 async fn pending_step_returns_terminal_record_in_invocation() {
4022 let (events, _guard) = crate::driver::test_support::EventMessages::install();
4023 let client = Arc::new(RefreshClient::new(|req| {
4024 let mut ops = Vec::new();
4025 if req.empty_number == 1 {
4026 ops.push(
4027 operation_like(&flaky_update(), OperationStatus::Succeeded)
4028 .step_details(StepDetails::builder().attempt(2).result("5").build())
4029 .build()
4030 .expect("all required fields set"),
4031 );
4032 }
4033 Ok(ops)
4034 }));
4035 let ctx = pending_flaky_ctx(&client);
4036 let runs = Arc::new(AtomicUsize::new(0));
4037 let body_runs = Arc::clone(&runs);
4038 let outcome = drive_handler(ctx, None, move |ctx| async move {
4039 let mut flaky = std::pin::pin!(
4040 ctx.step(move |_| async move {
4041 body_runs.fetch_add(1, Ordering::SeqCst);
4042 Ok::<u32, crate::BoxError>(7)
4043 })
4044 .name("flaky")
4045 .spawn()
4046 );
4047 poll_once(flaky.as_mut()).await;
4048 sleeping_step(&ctx).await?;
4049 let value = flaky.await.map_err(wire_err)?;
4050 Ok(value.to_string())
4051 })
4052 .await
4053 .expect("no panic");
4054 assert_eq!(outcome, InvocationOutcome::Complete("5".to_owned()));
4055 assert_eq!(runs.load(Ordering::SeqCst), 0, "the body never ran");
4056 assert_eq!(
4057 events.replayed(),
4058 0,
4059 "a resumed record emits no replay event"
4060 );
4061 }
4062
4063 #[tokio::test(start_paused = true)]
4067 async fn pending_step_returns_failed_record_in_invocation() {
4068 let (events, _guard) = crate::driver::test_support::EventMessages::install();
4069 let client = Arc::new(RefreshClient::new(|req| {
4070 let mut ops = Vec::new();
4071 if req.empty_number == 1 {
4072 ops.push(
4073 operation_like(&flaky_update(), OperationStatus::Failed)
4074 .step_details(
4075 StepDetails::builder()
4076 .attempt(2)
4077 .error(
4078 ErrorObject::builder()
4079 .error_type("PaymentError")
4080 .error_message("declined")
4081 .build(),
4082 )
4083 .build(),
4084 )
4085 .build()
4086 .expect("all required fields set"),
4087 );
4088 }
4089 Ok(ops)
4090 }));
4091 let ctx = pending_flaky_ctx(&client);
4092 let runs = Arc::new(AtomicUsize::new(0));
4093 let body_runs = Arc::clone(&runs);
4094 let outcome = drive_handler(ctx, None, move |ctx| async move {
4095 let mut flaky = std::pin::pin!(
4096 ctx.step(move |_| async move {
4097 body_runs.fetch_add(1, Ordering::SeqCst);
4098 Ok::<u32, crate::BoxError>(7)
4099 })
4100 .name("flaky")
4101 .spawn()
4102 );
4103 poll_once(flaky.as_mut()).await;
4104 sleeping_step(&ctx).await?;
4105 let err = flaky.await.expect_err("the recorded failure returns");
4106 let error_type = err
4107 .wire()
4108 .and_then(crate::WireError::error_type)
4109 .unwrap_or_default()
4110 .to_owned();
4111 Ok(error_type)
4112 })
4113 .await
4114 .expect("no panic");
4115 assert_eq!(
4116 outcome,
4117 InvocationOutcome::Complete("PaymentError".to_owned())
4118 );
4119 assert_eq!(runs.load(Ordering::SeqCst), 0, "the body never ran");
4120 assert_eq!(
4121 events.replayed(),
4122 0,
4123 "a resumed record emits no replay event"
4124 );
4125 }
4126
4127 fn flaky_update() -> OperationUpdate {
4129 OperationUpdate::builder()
4130 .id(crate::engine::compute_wire_id_public("1"))
4131 .r#type(OperationType::Step)
4132 .sub_type(super::STEP_SUB_TYPE)
4133 .name("flaky")
4134 .action(OperationAction::Start)
4135 .build()
4136 .expect("all required fields set")
4137 }
4138}
4139
4140#[cfg(test)]
4144mod start_write_tests {
4145 use crate::StepSemantics;
4146 use crate::driver::InvocationOutcome;
4147 use crate::driver::test_support::{LatencyClient, drive_handler, latency_ctx, wire_err};
4148 use aws_sdk_lambda::types::OperationAction;
4149 use std::sync::atomic::{AtomicUsize, Ordering};
4150 use std::sync::{Arc, Mutex};
4151 use std::time::Duration;
4152
4153 const LATENCY: Duration = Duration::from_millis(500);
4154
4155 fn ms(n: u64) -> Duration {
4156 Duration::from_millis(n)
4157 }
4158
4159 async fn timed_step(
4164 buffer_window: Duration,
4165 semantics: StepSemantics,
4166 ) -> (Arc<LatencyClient>, Duration, InvocationOutcome) {
4167 let client = Arc::new(LatencyClient::new(LATENCY));
4168 let ctx = latency_ctx(&client, buffer_window);
4169 let started = Arc::new(Mutex::new(None));
4170 let body_started = Arc::clone(&started);
4171 let clock = Arc::clone(&client);
4172 let outcome = drive_handler(ctx, None, move |ctx| async move {
4173 let body_clock = Arc::clone(&clock);
4174 let value = ctx
4175 .step(move |_| {
4176 let started = Arc::clone(&body_started);
4177 let clock = Arc::clone(&body_clock);
4178 async move {
4179 *started.lock().expect("lock") = Some(clock.elapsed());
4180 Ok(1_u32)
4181 }
4182 })
4183 .name("s")
4184 .semantics(semantics)
4185 .await
4186 .map_err(wire_err)?;
4187 Ok(format!("{value}@{}", clock.elapsed().as_millis()))
4188 })
4189 .await
4190 .expect("no panic");
4191 let started = started.lock().expect("lock").expect("the body ran");
4192 (client, started, outcome)
4193 }
4194
4195 #[tokio::test(start_paused = true)]
4196 async fn at_least_once_body_starts_before_start_response() {
4197 let (client, started, outcome) =
4198 timed_step(Duration::ZERO, StepSemantics::AtLeastOncePerRetry).await;
4199 assert_eq!(started, ms(0), "the body starts before START is answered");
4200 let start = client.request_with(&OperationAction::Start).expect("START");
4201 let succeed = client
4202 .request_with(&OperationAction::Succeed)
4203 .expect("SUCCEED");
4204 assert_eq!(start.received, ms(0));
4205 assert_eq!(succeed.received, ms(500), "SUCCEED waits for START");
4206 assert_eq!(outcome, InvocationOutcome::Complete("1@1000".to_owned()));
4207 }
4208
4209 #[tokio::test(start_paused = true)]
4210 async fn buffered_start_precedes_outcome() {
4211 let (client, started, outcome) =
4212 timed_step(ms(50), StepSemantics::AtLeastOncePerRetry).await;
4213 assert_eq!(started, ms(0), "the body starts before START is answered");
4214 let requests = client.requests();
4215 let first = requests.first().expect("a request");
4216 let actions: Vec<&OperationAction> = first
4217 .updates
4218 .iter()
4219 .map(aws_sdk_lambda::types::OperationUpdate::action)
4220 .collect();
4221 assert_eq!(
4222 actions,
4223 vec![&OperationAction::Start],
4224 "the first request carries only START"
4225 );
4226 assert!(
4227 requests.iter().skip(1).any(|r| r
4228 .updates
4229 .iter()
4230 .any(|u| u.action() == &OperationAction::Succeed)),
4231 "SUCCEED is in a later request"
4232 );
4233 assert!(matches!(outcome, InvocationOutcome::Complete(ref v) if v.starts_with("1@")));
4234 }
4235
4236 #[tokio::test(start_paused = true)]
4237 async fn at_most_once_body_waits_for_start_response() {
4238 let (_client, started, outcome) =
4239 timed_step(Duration::ZERO, StepSemantics::AtMostOncePerRetry).await;
4240 assert_eq!(started, ms(500), "the body starts after START is answered");
4241 assert!(matches!(outcome, InvocationOutcome::Complete(ref v) if v.starts_with("1@")));
4242 }
4243
4244 fn assert_failed_without_outcome(client: &LatencyClient, outcome: &InvocationOutcome) {
4247 let InvocationOutcome::Failed { error } = outcome else {
4248 unreachable!("expected a failed execution, got {outcome:?}");
4249 };
4250 assert_eq!(
4251 error.error_type(),
4252 Some(crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
4253 );
4254 assert!(client.request_with(&OperationAction::Start).is_some());
4255 assert!(!client.received_outcome(), "no outcome update was sent");
4256 }
4257
4258 #[tokio::test(start_paused = true)]
4259 async fn failed_start_write_writes_no_outcome() {
4260 let client = Arc::new(LatencyClient::new(LATENCY).failing_start());
4262 let runs = Arc::new(AtomicUsize::new(0));
4263 let body_runs = Arc::clone(&runs);
4264 let outcome = drive_handler(
4265 latency_ctx(&client, Duration::ZERO),
4266 None,
4267 move |ctx| async move {
4268 ctx.step(move |_| async move {
4269 body_runs.fetch_add(1, Ordering::SeqCst);
4270 Ok(1_u32)
4271 })
4272 .name("s")
4273 .await
4274 .map_err(wire_err)?;
4275 Ok("done".to_owned())
4276 },
4277 )
4278 .await
4279 .expect("no panic");
4280 assert_eq!(runs.load(Ordering::SeqCst), 1, "the body ran once");
4281 assert_failed_without_outcome(&client, &outcome);
4282
4283 let client = Arc::new(LatencyClient::new(LATENCY).failing_start());
4285 let checks = Arc::new(AtomicUsize::new(0));
4286 let check_runs = Arc::clone(&checks);
4287 let outcome = drive_handler(
4288 latency_ctx(&client, Duration::ZERO),
4289 None,
4290 move |ctx| async move {
4291 ctx.wait_for_condition(
4292 move |_, state: u32| {
4293 let checks = Arc::clone(&check_runs);
4294 async move {
4295 checks.fetch_add(1, Ordering::SeqCst);
4296 Ok(state + 1)
4297 }
4298 },
4299 0,
4300 )
4301 .wait_strategy_fn(|_state, _attempt| {
4302 crate::builders::wait_for_condition::WaitDecision::complete()
4303 })
4304 .name("c")
4305 .await
4306 .map_err(wire_err)?;
4307 Ok("done".to_owned())
4308 },
4309 )
4310 .await
4311 .expect("no panic");
4312 assert_eq!(checks.load(Ordering::SeqCst), 1, "the check ran once");
4313 assert_failed_without_outcome(&client, &outcome);
4314 }
4315
4316 fn assert_start_kept(
4321 client: &LatencyClient,
4322 ctx: &crate::DurableContext,
4323 result: Result<InvocationOutcome, Box<dyn std::any::Any + Send + 'static>>,
4324 returned: Duration,
4325 ) {
4326 let start = client.request_with(&OperationAction::Start).expect("START");
4327 let answered = start.answered.expect("START was answered");
4328 assert_eq!(answered, start.received + LATENCY);
4329 assert!(!client.received_outcome(), "no outcome update was sent");
4330 let wire_id = start.updates.first().expect("an update").id();
4331 assert_eq!(crate::engine::compute_wire_id_public("1"), wire_id);
4332 let view = ctx.checkpoint_status_view("1").expect("a record");
4333 assert_eq!(view.status, crate::engine::CheckpointStatus::Started);
4334 let payload = result.expect_err("the panic leaves the drive");
4335 assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
4336 assert!(
4337 returned >= answered,
4338 "the drive returned at {returned:?}, before {answered:?}"
4339 );
4340 }
4341
4342 async fn panicking_step(buffer_window: Duration) {
4346 let client = Arc::new(LatencyClient::new(LATENCY));
4347 let ctx = latency_ctx(&client, buffer_window);
4348 let clock = Arc::clone(&client);
4349 let result = drive_handler(ctx.clone(), None, |ctx| async move {
4350 #[expect(clippy::panic)] let (): () = ctx
4352 .step(move |_| async move {
4353 assert_eq!(
4354 clock.elapsed(),
4355 ms(0),
4356 "the body runs before START is answered"
4357 );
4358 panic!("boom")
4359 })
4360 .name("s")
4361 .await
4362 .map_err(wire_err)?;
4363 Ok("done".to_owned())
4364 })
4365 .await;
4366 assert_start_kept(&client, &ctx, result, client.elapsed());
4367 }
4368
4369 #[tokio::test(start_paused = true)]
4370 async fn body_panic_before_start_response_keeps_start_write() {
4371 panicking_step(Duration::ZERO).await;
4373 panicking_step(ms(50)).await;
4375
4376 let client = Arc::new(LatencyClient::new(LATENCY));
4378 let ctx = latency_ctx(&client, Duration::ZERO);
4379 let clock = Arc::clone(&client);
4380 let result = drive_handler(ctx.clone(), None, |ctx| async move {
4381 #[expect(clippy::panic)] ctx.wait_for_condition(
4383 move |_, _state: u32| {
4384 let clock = Arc::clone(&clock);
4385 async move {
4386 assert_eq!(
4387 clock.elapsed(),
4388 ms(0),
4389 "the check runs before START is answered"
4390 );
4391 panic!("boom")
4392 }
4393 },
4394 0,
4395 )
4396 .wait_strategy_fn(|_state, _attempt| {
4397 crate::builders::wait_for_condition::WaitDecision::complete()
4398 })
4399 .name("c")
4400 .await
4401 .map_err(wire_err)?;
4402 Ok("done".to_owned())
4403 })
4404 .await;
4405 assert_start_kept(&client, &ctx, result, client.elapsed());
4406 }
4407}