1#![cfg_attr(test, allow(clippy::unwrap_used))]
112#![cfg_attr(test, allow(clippy::expect_used))]
113
114#[cfg(doctest)]
117#[doc = include_str!("../README.md")]
118mod readme_doctests {}
119
120pub mod builders;
121pub(crate) mod callback;
122pub(crate) mod checkpoint_coalescer;
123pub(crate) mod child;
124pub(crate) mod client;
125pub(crate) mod combinator;
126mod context;
127pub(crate) mod coordinator;
128pub(crate) mod driver;
129mod engine;
130mod error;
131mod future;
132pub(crate) mod invoke;
133pub(crate) mod map_parallel; pub mod observability;
135mod operation;
136mod operation_sub_type;
137mod options;
138pub mod plugin;
139pub mod serdes;
140pub(crate) mod start;
141pub(crate) mod step;
142mod task;
143#[cfg(feature = "test-util")]
144pub mod test_util;
145pub(crate) mod tracing_layer;
146pub(crate) mod wait;
147pub(crate) mod wait_for_condition;
148pub(crate) mod with_retry;
149
150pub use self::tracing_layer::ReplayFilterLayer;
151
152pub use self::client::{
153 CheckpointOutput, ExecutionClient, ExecutionClientError, ExecutionClientErrorScope,
154 GetStateOutput, LambdaExecutionClient,
155};
156pub use self::context::{DurableContext, StepContext};
157pub use self::error::{
158 CallbackError, CallbackErrorKind, ChildContextError, ChildContextErrorKind, CombinatorError,
159 CombinatorErrorKind, InvokeError, InvokeErrorKind, JoinFailed, MaxChecksExceeded,
160 NonDeterministicExecutionError, NonDeterministicExecutionErrorKind, OperationError,
161 OperationErrorKind, OperationMismatch, ReplayedFailure, StepError, StepErrorKind, TypedError,
162 UnexpectedStatus, UnrecognizedStatus, WaitError, WaitErrorKind, WaitForConditionError,
163 WaitForConditionErrorKind, WireError,
164};
165pub use self::future::{Branch, BranchHandle, BranchKey, BranchSet, BranchValue, DurableFuture};
166pub use self::operation::{InvocationStatus, OperationKind, OperationStatus};
167pub use self::operation_sub_type::{InvalidOperationSubType, OperationSubType};
168pub use self::options::{Options, OptionsBuilder, OptionsValidationError};
169pub use self::serdes::{JsonSerdes, PassthroughSerdes, PayloadSerializer, Serdes};
170pub use self::step::StepSemantics;
171
172pub use aws_config::SdkConfig;
174pub use aws_sdk_lambda::Client as LambdaClient;
175pub use lambda_runtime::{self, Context as LambdaContext};
176
177pub mod lambda_types {
199 pub use aws_sdk_lambda::primitives::DateTime;
200 pub use aws_sdk_lambda::types::{
201 CallbackDetails, CallbackOptions, ChainedInvokeDetails, ChainedInvokeOptions,
202 ContextDetails, ContextOptions, ErrorObject, ExecutionDetails, Operation, OperationAction,
203 OperationStatus, OperationType, OperationUpdate, StepDetails, StepOptions, WaitDetails,
204 WaitOptions,
205 };
206 #[cfg(feature = "test-util")]
207 pub use aws_sdk_lambda::types::{
208 CallbackFailedDetails, CallbackStartedDetails, CallbackSucceededDetails,
209 CallbackTimedOutDetails, ChainedInvokeFailedDetails, ChainedInvokeStartedDetails,
210 ChainedInvokeStoppedDetails, ChainedInvokeSucceededDetails, ChainedInvokeTimedOutDetails,
211 ContextFailedDetails, ContextStartedDetails, ContextSucceededDetails, Event, EventError,
212 EventInput, EventResult, EventType, ExecutionFailedDetails, ExecutionStartedDetails,
213 ExecutionStoppedDetails, ExecutionSucceededDetails, ExecutionTimedOutDetails,
214 InvocationCompletedDetails, RetryDetails, StepFailedDetails, StepStartedDetails,
215 StepSucceededDetails, WaitCancelledDetails, WaitStartedDetails, WaitSucceededDetails,
216 };
217}
218
219use serde::{Deserialize, Serialize};
220use std::future::Future;
221use std::time::Duration;
222
223pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
241
242pub fn init_default_subscriber() -> Result<(), BoxError> {
296 init_default_subscriber_with_writer(std::io::stdout)
297}
298
299pub fn init_default_subscriber_with_writer<W>(writer: W) -> Result<(), BoxError>
318where
319 W: for<'w> lambda_runtime::tracing::subscriber::fmt::MakeWriter<'w> + Send + Sync + 'static,
320{
321 use lambda_runtime::tracing::subscriber::util::SubscriberInitExt as _;
322
323 let settings = tracing_layer::LogSettings::from_env();
324 tracing_layer::default_subscriber(
325 &settings.level,
326 &settings.directives,
327 &settings.format,
328 writer,
329 )
330 .try_init()?;
331 Ok(())
332}
333
334#[derive(Debug, Clone, PartialEq, Eq)]
365#[non_exhaustive]
366pub enum RetryDecision {
367 #[non_exhaustive]
369 Retry {
370 delay: Duration,
372 },
373 Stop,
375}
376
377impl RetryDecision {
378 #[must_use]
381 pub fn retry(delay: Duration) -> Self {
382 Self::Retry { delay }
383 }
384}
385
386pub(crate) type RetryStrategy = Box<dyn Fn(&StepError, u32) -> RetryDecision + Send + Sync>;
395
396pub async fn run<F, E, Fut, O>(handler: F) -> Result<(), lambda_runtime::Error>
465where
466 F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
467 E: for<'de> Deserialize<'de> + Send + 'static,
468 Fut: Future<Output = Result<O, BoxError>> + Send,
469 O: Serialize + Send + 'static,
470{
471 run_with_options(handler, Options::default()).await
472}
473
474pub async fn run_with_options<F, E, Fut, O>(
520 handler: F,
521 options: Options,
522) -> Result<(), lambda_runtime::Error>
523where
524 F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
525 E: for<'de> Deserialize<'de> + Send + 'static,
526 Fut: Future<Output = Result<O, BoxError>> + Send,
527 O: Serialize + Send + 'static,
528{
529 lambda_runtime::run(lambda_runtime::service_fn(wrap(handler, options))).await
530}
531
532#[derive(Debug)]
534struct InvocationEnvelope {
535 execution_arn: String,
536 checkpoint_token: String,
537 updated_operation_ids: Vec<String>,
541}
542
543fn has_envelope_shape(payload: &serde_json::Value) -> bool {
549 payload.get("DurableExecutionArn").is_some()
550 || payload.get("CheckpointToken").is_some()
551 || payload.get("InitialExecutionState").is_some()
552}
553
554fn parse_envelope(
565 payload: &serde_json::Value,
566) -> Result<Option<InvocationEnvelope>, lambda_runtime::Error> {
567 if !has_envelope_shape(payload) {
568 return Ok(None);
569 }
570
571 let execution_arn = match payload.get("DurableExecutionArn") {
572 Some(v) => v
573 .as_str()
574 .ok_or_else(|| {
575 lambda_runtime::Error::from(
576 "malformed invocation envelope: \"DurableExecutionArn\" is present but is not \
577 a string"
578 .to_owned(),
579 )
580 })?
581 .to_owned(),
582 None => {
583 return Err(lambda_runtime::Error::from(
584 "malformed invocation envelope: required field \"DurableExecutionArn\" is missing"
585 .to_owned(),
586 ));
587 }
588 };
589
590 let checkpoint_token = match payload.get("CheckpointToken") {
591 Some(v) => v
592 .as_str()
593 .ok_or_else(|| {
594 lambda_runtime::Error::from(
595 "malformed invocation envelope: \"CheckpointToken\" is present but is not a \
596 string"
597 .to_owned(),
598 )
599 })?
600 .to_owned(),
601 None => {
602 return Err(lambda_runtime::Error::from(
603 "malformed invocation envelope: required field \"CheckpointToken\" is missing"
604 .to_owned(),
605 ));
606 }
607 };
608
609 let updated_operation_ids = payload
610 .get("UpdatedOperationIds")
611 .and_then(serde_json::Value::as_array)
612 .map(|ids| {
613 ids.iter()
614 .filter_map(serde_json::Value::as_str)
615 .map(str::to_owned)
616 .collect()
617 })
618 .unwrap_or_default();
619
620 Ok(Some(InvocationEnvelope {
621 execution_arn,
622 checkpoint_token,
623 updated_operation_ids,
624 }))
625}
626
627fn inline_input_payload(
643 payload: &serde_json::Value,
644) -> Result<client::PageInput, lambda_runtime::Error> {
645 let execution_op = payload
646 .get("InitialExecutionState")
647 .and_then(|s| s.get("Operations"))
648 .and_then(serde_json::Value::as_array)
649 .and_then(|ops| {
650 ops.iter().find(|op| {
651 op.get("Type")
652 .and_then(serde_json::Value::as_str)
653 .is_some_and(|t| t.eq_ignore_ascii_case("EXECUTION"))
654 })
655 });
656 let Some(op) = execution_op else {
657 return Ok(client::PageInput::NoExecutionOperation);
658 };
659 match op
660 .get("ExecutionDetails")
661 .and_then(|d| d.get("InputPayload"))
662 {
663 None | Some(serde_json::Value::Null) => Ok(client::PageInput::Execution(None)),
664 Some(serde_json::Value::String(value)) => {
665 Ok(client::PageInput::Execution(Some(value.clone())))
666 }
667 Some(_) => Err(lambda_runtime::Error::from(
668 "malformed invocation envelope: InputPayload is not a string".to_owned(),
669 )),
670 }
671}
672
673fn extract_customer_input<E>(input_payload: Option<&str>) -> Result<E, lambda_runtime::Error>
688where
689 E: for<'de> Deserialize<'de>,
690{
691 serde_json::from_str(input_payload.unwrap_or("{}"))
692 .map_err(|e| lambda_runtime::Error::from(format!("deserialize customer input: {e}")))
693}
694
695fn wire_error_from_box_error(err: BoxError) -> WireError {
704 match err.downcast::<OperationError>() {
705 Ok(op_err) => wire_error_from_operation_error(&op_err),
706 Err(other) => error::wire_error_for(&*other, "Error"),
707 }
708}
709
710fn serialize_handler_result<T: Serialize>(result: &T) -> Result<String, WireError> {
716 serde_json::to_string(result).map_err(|e| error::wire_error_for(&e, "Error"))
717}
718
719fn wire_error_from_operation_error(err: &OperationError) -> WireError {
729 if let OperationErrorKind::Callback(cb_err) = err.kind()
730 && matches!(cb_err.kind(), CallbackErrorKind::ExternalFailure)
731 && let Some(wire) = err.wire()
732 {
733 return wire.clone();
735 }
736 error::wire_error_for(err, err.kind().wire_type_name())
737}
738
739fn parse_stack_trace(error: &serde_json::Value) -> Option<Vec<String>> {
741 let frames: Vec<String> = error
742 .get("StackTrace")?
743 .as_array()?
744 .iter()
745 .filter_map(serde_json::Value::as_str)
746 .map(String::from)
747 .collect();
748 if frames.is_empty() {
749 None
750 } else {
751 Some(frames)
752 }
753}
754
755fn parse_inline_operations(payload: &serde_json::Value) -> (engine::CheckpointLog, Option<String>) {
765 let initial_state = payload.get("InitialExecutionState");
766
767 let next_marker = initial_state
773 .and_then(|s| s.get("NextMarker"))
774 .and_then(serde_json::Value::as_str)
775 .filter(|m| !m.is_empty())
776 .map(String::from);
777
778 let records: Vec<(String, engine::CheckpointRecord)> = initial_state
782 .and_then(|s| s.get("Operations"))
783 .and_then(serde_json::Value::as_array)
784 .map(|ops| ops.iter().filter_map(parse_single_operation).collect())
785 .unwrap_or_default();
786
787 (engine::CheckpointLog::from_records(records), next_marker)
788}
789
790pub(crate) fn parse_envelope_timestamp(value: &serde_json::Value) -> Option<std::time::SystemTime> {
797 use aws_sdk_lambda::primitives::{DateTime, DateTimeFormat};
798 let parsed = match value {
799 serde_json::Value::String(s) => DateTime::from_str(s, DateTimeFormat::DateTime).ok()?,
800 serde_json::Value::Number(n) => DateTime::from_millis(n.as_i64()?),
801 _ => return None,
802 };
803 to_system_time(parsed)
804}
805
806pub(crate) fn to_system_time<T: TryInto<std::time::SystemTime>>(
810 timestamp: T,
811) -> Option<std::time::SystemTime> {
812 timestamp.try_into().ok()
813}
814
815fn parse_inline_status(status_str: &str) -> engine::CheckpointStatus {
818 match status_str.to_ascii_uppercase().as_str() {
819 "STARTED" => engine::CheckpointStatus::Started,
820 "SUCCEEDED" => engine::CheckpointStatus::Succeeded,
821 "FAILED" => engine::CheckpointStatus::Failed,
822 "PENDING" => engine::CheckpointStatus::Pending,
823 "READY" => engine::CheckpointStatus::Ready,
824 "CANCELLED" => engine::CheckpointStatus::Cancelled,
825 "TIMED_OUT" => engine::CheckpointStatus::TimedOut,
826 "STOPPED" => engine::CheckpointStatus::Stopped,
827 _ => engine::CheckpointStatus::Unknown(status_str.to_owned()),
828 }
829}
830
831fn execution_info_from_json(op: &serde_json::Value) -> Option<plugin::OperationInfo> {
837 let id = op.get("Id").and_then(serde_json::Value::as_str)?;
838 let op_type = op.get("Type").and_then(serde_json::Value::as_str)?;
839 let status = parse_inline_status(
840 op.get("Status")
841 .and_then(serde_json::Value::as_str)
842 .unwrap_or("STARTED"),
843 );
844 plugin::OperationInfo::execution(
845 id,
846 op_type,
847 &status,
848 op.get("ParentId")
849 .and_then(serde_json::Value::as_str)
850 .map(String::from),
851 op.get("StartTimestamp").and_then(parse_envelope_timestamp),
852 op.get("EndTimestamp").and_then(parse_envelope_timestamp),
853 )
854}
855
856fn inline_bootstrap_state(
860 payload: &serde_json::Value,
861) -> Result<(client::BootstrapState, Option<String>), lambda_runtime::Error> {
862 let input_payload = inline_input_payload(payload)?.into_payload();
863 let (log, next_marker) = parse_inline_operations(payload);
864 let state = client::BootstrapState {
865 log,
866 input_payload,
867 execution: inline_execution_info(payload),
868 };
869 Ok((state, next_marker))
870}
871
872fn inline_execution_info(payload: &serde_json::Value) -> Option<plugin::OperationInfo> {
875 payload
876 .get("InitialExecutionState")
877 .and_then(|s| s.get("Operations"))
878 .and_then(serde_json::Value::as_array)
879 .and_then(|ops| {
880 ops.iter().find(|op| {
881 op.get("Type")
882 .and_then(serde_json::Value::as_str)
883 .is_some_and(|t| t.eq_ignore_ascii_case("EXECUTION"))
884 })
885 })
886 .and_then(execution_info_from_json)
887}
888
889#[expect(clippy::too_many_lines)] fn parse_single_operation(op: &serde_json::Value) -> Option<(String, engine::CheckpointRecord)> {
892 let id = op.get("Id").and_then(serde_json::Value::as_str)?;
893 let op_type = op.get("Type").and_then(serde_json::Value::as_str)?;
894 if op_type.eq_ignore_ascii_case("Execution") {
896 return None;
897 }
898 let status_str = op
899 .get("Status")
900 .and_then(serde_json::Value::as_str)
901 .unwrap_or("STARTED");
902 let status = parse_inline_status(status_str);
908
909 let step_details = op.get("StepDetails");
911 let result = step_details
912 .and_then(|d| d.get("Result"))
913 .and_then(serde_json::Value::as_str)
914 .map(String::from);
915 let error = step_details.and_then(|d| d.get("Error"));
916 let error_type = error
917 .and_then(|e| e.get("ErrorType"))
918 .and_then(serde_json::Value::as_str)
919 .map(String::from);
920 let error_message = error
921 .and_then(|e| e.get("ErrorMessage"))
922 .and_then(serde_json::Value::as_str)
923 .map(String::from);
924 let error_data = error
925 .and_then(|e| e.get("ErrorData"))
926 .and_then(serde_json::Value::as_str)
927 .map(String::from);
928 let stack_trace = error.and_then(parse_stack_trace);
929 #[expect(clippy::cast_possible_truncation)] #[expect(clippy::cast_sign_loss)] let attempt = step_details
932 .and_then(|d| d.get("Attempt"))
933 .and_then(serde_json::Value::as_i64)
934 .map_or(0, |a| a.clamp(0, i64::from(u32::MAX)) as u32);
935
936 let due_at = op
938 .get("WaitDetails")
939 .and_then(|d| d.get("ScheduledEndTimestamp"))
940 .or_else(|| step_details.and_then(|d| d.get("NextAttemptTimestamp")))
941 .and_then(parse_envelope_timestamp);
942
943 let invoke_details = op.get("ChainedInvokeDetails");
945 let invoke_result = invoke_details
946 .and_then(|d| d.get("Result"))
947 .and_then(serde_json::Value::as_str)
948 .map(String::from);
949 let invoke_error = invoke_details.and_then(|d| d.get("Error"));
950 let invoke_error_type = invoke_error
951 .and_then(|e| e.get("ErrorType"))
952 .and_then(serde_json::Value::as_str)
953 .map(String::from);
954 let invoke_error_message = invoke_error
955 .and_then(|e| e.get("ErrorMessage"))
956 .and_then(serde_json::Value::as_str)
957 .map(String::from);
958 let invoke_error_data = invoke_error
959 .and_then(|e| e.get("ErrorData"))
960 .and_then(serde_json::Value::as_str)
961 .map(String::from);
962 let invoke_stack_trace = invoke_error.and_then(parse_stack_trace);
963
964 let context_details = op.get("ContextDetails");
966 let replay_children = context_details
967 .and_then(|d| d.get("ReplayChildren"))
968 .and_then(serde_json::Value::as_bool)
969 .unwrap_or(false);
970
971 let callback_details = op.get("CallbackDetails");
973 let callback_id = callback_details
974 .and_then(|d| d.get("CallbackId"))
975 .and_then(serde_json::Value::as_str)
976 .map(String::from);
977
978 let result = result.or_else(|| {
980 context_details
981 .and_then(|d| d.get("Result"))
982 .and_then(serde_json::Value::as_str)
983 .map(String::from)
984 });
985
986 let context_error = context_details.and_then(|d| d.get("Error"));
988 let error_type = error_type.or_else(|| {
989 context_error
990 .and_then(|e| e.get("ErrorType"))
991 .and_then(serde_json::Value::as_str)
992 .map(String::from)
993 });
994 let error_message = error_message.or_else(|| {
995 context_error
996 .and_then(|e| e.get("ErrorMessage"))
997 .and_then(serde_json::Value::as_str)
998 .map(String::from)
999 });
1000 let error_data = error_data.or_else(|| {
1001 context_error
1002 .and_then(|e| e.get("ErrorData"))
1003 .and_then(serde_json::Value::as_str)
1004 .map(String::from)
1005 });
1006 let stack_trace = stack_trace.or_else(|| context_error.and_then(parse_stack_trace));
1007
1008 let result = result.or_else(|| {
1010 callback_details
1011 .and_then(|d| d.get("Result"))
1012 .and_then(serde_json::Value::as_str)
1013 .map(String::from)
1014 });
1015
1016 let callback_error = callback_details.and_then(|d| d.get("Error"));
1018 let error_type = error_type.or_else(|| {
1019 callback_error
1020 .and_then(|e| e.get("ErrorType"))
1021 .and_then(serde_json::Value::as_str)
1022 .map(String::from)
1023 });
1024 let error_message = error_message.or_else(|| {
1025 callback_error
1026 .and_then(|e| e.get("ErrorMessage"))
1027 .and_then(serde_json::Value::as_str)
1028 .map(String::from)
1029 });
1030 let error_data = error_data.or_else(|| {
1031 callback_error
1032 .and_then(|e| e.get("ErrorData"))
1033 .and_then(serde_json::Value::as_str)
1034 .map(String::from)
1035 });
1036 let stack_trace = stack_trace.or_else(|| callback_error.and_then(parse_stack_trace));
1037
1038 Some((
1039 id.to_owned(),
1040 engine::CheckpointRecord {
1041 id: id.to_owned(),
1042 status,
1043 result,
1044 error_type,
1045 error_message,
1046 error_data,
1047 stack_trace,
1048 attempt,
1049 invoke_result,
1050 invoke_error_type,
1051 invoke_error_message,
1052 invoke_error_data,
1053 invoke_stack_trace,
1054 replay_children,
1055 callback_id,
1056 op_type: Some(op_type.to_owned()),
1057 sub_type: op
1058 .get("SubType")
1059 .and_then(serde_json::Value::as_str)
1060 .map(String::from),
1061 op_name: op
1062 .get("Name")
1063 .and_then(serde_json::Value::as_str)
1064 .map(String::from),
1065 due_at,
1066 parent_id: op
1067 .get("ParentId")
1068 .and_then(serde_json::Value::as_str)
1069 .map(String::from),
1070 start_timestamp: op.get("StartTimestamp").and_then(parse_envelope_timestamp),
1071 end_timestamp: op.get("EndTimestamp").and_then(parse_envelope_timestamp),
1072 },
1073 ))
1074}
1075
1076#[derive(Debug, serde::Deserialize)]
1086pub struct InvocationPayload(pub(crate) serde_json::Value);
1087
1088#[derive(Debug, serde::Serialize)]
1097pub struct InvocationResponse(pub(crate) serde_json::Value);
1098
1099pub(crate) type BoxedInvocationFuture = std::pin::Pin<
1105 Box<dyn Future<Output = Result<InvocationResponse, lambda_runtime::Error>> + Send>,
1106>;
1107
1108pub fn wrap<F, E, Fut, O>(
1153 handler: F,
1154 options: Options,
1155) -> impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> BoxedInvocationFuture + Send + Sync
1156where
1157 F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
1158 E: for<'de> Deserialize<'de> + Send + 'static,
1159 Fut: Future<Output = Result<O, BoxError>> + Send,
1160 O: Serialize + Send + 'static,
1161{
1162 use std::sync::Arc as StdArc;
1163
1164 let Options {
1168 sdk_config,
1169 lambda_client,
1170 execution_client,
1171 checkpoint_delay,
1172 plugins,
1173 plugin_child_operations_depth,
1174 } = options;
1175 let checkpoint_buffer_window = checkpoint_delay.unwrap_or(Duration::ZERO);
1179 let preset_client: Option<StdArc<dyn ExecutionClient>> = execution_client.or_else(|| {
1182 base_lambda_client_from_options(sdk_config, lambda_client)
1183 .map(|c| StdArc::new(LambdaExecutionClient::new(c)) as StdArc<dyn ExecutionClient>)
1184 });
1185 wrap_with_provider(
1186 handler,
1187 ClientProvider::new(preset_client),
1188 checkpoint_buffer_window,
1189 plugin_child_operations_depth,
1190 plugins.into(),
1191 )
1192}
1193
1194#[expect(clippy::too_many_lines)] fn wrap_with_provider<F, E, Fut, O>(
1201 handler: F,
1202 provider: ClientProvider,
1203 checkpoint_buffer_window: Duration,
1204 plugin_child_operations_depth: u32,
1205 plugin_factories: std::sync::Arc<plugin::Factories>,
1206) -> impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> BoxedInvocationFuture + Send + Sync
1207where
1208 F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
1209 E: for<'de> Deserialize<'de> + Send + 'static,
1210 Fut: Future<Output = Result<O, BoxError>> + Send,
1211 O: Serialize + Send + 'static,
1212{
1213 use std::sync::Arc as StdArc;
1214
1215 let handler = StdArc::new(handler);
1216 let provider = StdArc::new(provider);
1217
1218 move |event: lambda_runtime::LambdaEvent<InvocationPayload>| -> std::pin::Pin<
1219 Box<dyn Future<Output = Result<InvocationResponse, lambda_runtime::Error>> + Send>,
1220 > {
1221 let handler = StdArc::clone(&handler);
1222 let provider = StdArc::clone(&provider);
1223 let plugin_factories = StdArc::clone(&plugin_factories);
1224 Box::pin(async move {
1225 let (InvocationPayload(raw_payload), lambda_ctx) = event.into_parts();
1226
1227 let envelope = parse_envelope(&raw_payload)?.ok_or_else(|| {
1229 lambda_runtime::Error::from(
1230 "invocation payload is not a durable execution envelope \
1231 (missing DurableExecutionArn, CheckpointToken, and \
1232 InitialExecutionState)"
1233 .to_owned(),
1234 )
1235 })?;
1236 let execution_arn = envelope.execution_arn;
1237 let checkpoint_token = envelope.checkpoint_token;
1238 let updated_operation_ids = envelope.updated_operation_ids;
1239
1240 let (first_page, initial_marker) = inline_bootstrap_state(&raw_payload)?;
1243
1244 let exec_client = provider.get().await;
1248
1249 let (checkpoint_log, input_payload, execution) = match client::resolve_bootstrap_log(
1255 exec_client.as_ref(),
1256 &execution_arn,
1257 &checkpoint_token,
1258 first_page,
1259 initial_marker.as_deref(),
1260 )
1261 .await
1262 {
1263 Ok(state) => (StdArc::new(state.log), state.input_payload, state.execution),
1264 Err(e) if e.is_retryable() => {
1265 return Err(lambda_runtime::Error::from(format!(
1266 "failed to paginate initial state: {e}"
1267 )));
1268 }
1269 Err(e) => {
1270 return Ok(failed_envelope(&WireError::new(
1271 Some(client::bootstrap_error_type(&e)),
1272 Some(client::client_error_record_message(&e)),
1273 )));
1274 }
1275 };
1276
1277 let customer_input: E = extract_customer_input(input_payload.as_deref())?;
1280
1281 let plugins = create_invocation_plugins(
1287 &plugin_factories,
1288 &lambda_ctx,
1289 &execution_arn,
1290 input_payload.as_deref(),
1291 &checkpoint_log,
1292 execution.as_ref(),
1293 updated_operation_ids,
1294 );
1295 let _shut_down = plugins.as_ref().map(plugin::ShutDownOnDrop::new);
1298
1299 if let Some(plugins) = &plugins {
1302 plugins.invocation_start().await;
1303 }
1304 let end_log = StdArc::clone(&checkpoint_log);
1306
1307 let ctx = DurableContext::new_root_with_client_and_defaults(
1308 execution_arn,
1309 lambda_ctx,
1310 checkpoint_log,
1311 exec_client,
1312 checkpoint_token,
1313 checkpoint_buffer_window,
1314 plugin_child_operations_depth,
1315 plugins.clone(),
1316 );
1317
1318 let flush_ctx = ctx.clone();
1321 let coordinator = StdArc::clone(ctx.coordinator());
1322 if let Some(plugins) = &plugins {
1327 plugins.attach(&coordinator);
1328 }
1329 let remaining = ctx.remaining_time();
1330 let replay_span = ctx.replay_span();
1331
1332 {
1339 let _execution_scope = replay_span.enter();
1340 tracing_layer::invocation_begin_event(
1341 ctx.is_replaying(),
1342 ctx.execution_arn(),
1343 &ctx.lambda_context().request_id,
1344 );
1345 }
1346
1347 let task_ownership = StdArc::clone(ctx.task_ownership());
1362 let root = context::run_body(context::ActiveBody::Context(String::new()), async move {
1363 match (handler)(customer_input, ctx).await {
1364 Ok(result) => serialize_handler_result(&result),
1365 Err(e) => Err(wire_error_from_box_error(e)),
1366 }
1367 });
1368 let frames = plugin::Frames::invocation(plugins.clone());
1388 let root_cx = task::TaskContext::new(frames.clone(), replay_span);
1389 let finished = plugin::Frames::instrument(
1390 &frames,
1391 CatchFinished::new(finish_invocation(
1392 root,
1393 root_cx,
1394 task_ownership,
1395 coordinator,
1396 &flush_ctx,
1397 remaining,
1398 plugins.as_deref(),
1399 &end_log,
1400 )),
1401 )
1402 .await;
1403 drop(frames);
1407 respond(finished, flush_ctx, plugins)
1408 })
1409 }
1410}
1411
1412#[cfg(test)]
1414mod test_hooks {
1415 tokio::task_local! {
1416 pub(crate) static PENDING_AFTER_DRIVE:
1420 std::sync::Arc<std::sync::atomic::AtomicBool>;
1421 }
1422
1423 pub(crate) async fn pending_once_after_drive() {
1424 let Ok(flag) = PENDING_AFTER_DRIVE.try_with(std::sync::Arc::clone) else {
1425 return;
1426 };
1427 let mut pending = true;
1428 std::future::poll_fn(|cx| {
1429 if std::mem::take(&mut pending) {
1430 flag.store(true, std::sync::atomic::Ordering::SeqCst);
1431 cx.waker().wake_by_ref();
1432 std::task::Poll::Pending
1433 } else {
1434 std::task::Poll::Ready(())
1435 }
1436 })
1437 .await;
1438 }
1439
1440 pub(crate) struct HeldGuard {
1442 pub(crate) reached: std::sync::Arc<std::sync::atomic::AtomicBool>,
1444 pub(crate) make: Box<dyn Fn() -> Box<dyn Send> + Send + Sync>,
1446 }
1447
1448 tokio::task_local! {
1449 pub(crate) static PENDING_HOLDING_GUARD: std::sync::Arc<HeldGuard>;
1454 }
1455
1456 pub(crate) async fn pending_holding_guard() {
1457 let Ok(held) = PENDING_HOLDING_GUARD.try_with(std::sync::Arc::clone) else {
1458 return;
1459 };
1460 let guard = (held.make)();
1461 let mut pending = true;
1462 std::future::poll_fn(|cx| {
1463 if std::mem::take(&mut pending) {
1464 held.reached
1465 .store(true, std::sync::atomic::Ordering::SeqCst);
1466 cx.waker().wake_by_ref();
1467 std::task::Poll::Pending
1468 } else {
1469 std::task::Poll::Ready(())
1470 }
1471 })
1472 .await;
1473 drop(guard);
1474 }
1475}
1476
1477enum Finished {
1479 Respond(
1481 Result<InvocationResponse, lambda_runtime::Error>,
1482 Option<Box<dyn std::any::Any + Send + 'static>>,
1483 ),
1484 Panicked(Box<dyn std::any::Any + Send + 'static>),
1486}
1487
1488#[expect(
1492 clippy::too_many_arguments,
1493 reason = "the root task's future and its context are passed apart, as `spawn_root` takes them"
1494)]
1495async fn finish_invocation<R>(
1496 root: R,
1497 root_cx: task::TaskContext,
1498 task_ownership: std::sync::Arc<driver::TaskOwnership>,
1499 coordinator: std::sync::Arc<coordinator::InvocationCoordinator>,
1500 flush_ctx: &DurableContext,
1501 remaining: Option<Duration>,
1502 plugins: Option<&plugin::InvocationPlugins>,
1503 end_log: &engine::CheckpointLog,
1504) -> Finished
1505where
1506 R: Future<Output = Result<String, WireError>> + Send + 'static,
1507{
1508 let (outcome, fatal_won) = match driver::drive_invocation(
1509 root,
1510 root_cx,
1511 task_ownership,
1512 coordinator,
1513 flush_ctx.clone(),
1514 remaining,
1515 )
1516 .await
1517 {
1518 Ok(driven) => driven,
1519 Err(payload) => {
1520 let end = InvocationEnd::panicked(&*payload);
1521 end_invocation(plugins, end_log, flush_ctx, end, None, false)
1525 .await
1526 .into_iter()
1527 .for_each(task::discard_panic);
1528 return Finished::Panicked(payload);
1529 }
1530 };
1531 #[cfg(test)]
1543 test_hooks::pending_once_after_drive().await;
1544 #[cfg(test)]
1545 test_hooks::pending_holding_guard().await;
1546
1547 if !matches!(outcome, driver::InvocationOutcome::Fault { .. }) {
1559 flush_ctx.flush_pending_checkpoints().await;
1560 }
1561 let outcome = match flush_ctx.coordinator().write_failure() {
1566 Some(failure) if !fatal_won => failure.into_outcome(),
1567 _ => outcome,
1568 };
1569
1570 let (response, end) = match outcome {
1599 driver::InvocationOutcome::Complete(serialized)
1600 if serialized.len() > LAMBDA_RESPONSE_SIZE_LIMIT =>
1601 {
1602 checkpoint_oversized_result(flush_ctx, serialized).await
1603 }
1604 outcome => outcome_envelope(outcome, flush_ctx),
1605 };
1606 let (response, end) = driver::withdrawal_answer(flush_ctx, response, end);
1607 let late = end_invocation(plugins, end_log, flush_ctx, end, Some(&response), fatal_won).await;
1608 Finished::Respond(response, late)
1609}
1610
1611fn respond(
1616 finished: Finished,
1617 ctx: DurableContext,
1618 plugins: Option<std::sync::Arc<plugin::InvocationPlugins>>,
1619) -> Result<InvocationResponse, lambda_runtime::Error> {
1620 let payload = match finished {
1621 Finished::Respond(response, None) => return response,
1622 Finished::Respond(response, Some(payload)) => {
1623 drop_contained(response);
1624 payload
1625 }
1626 Finished::Panicked(payload) => payload,
1627 };
1628 drop_contained(ctx);
1631 drop_contained(plugins);
1632 std::panic::resume_unwind(payload)
1633}
1634
1635fn drop_contained<T>(value: T) {
1637 if let Err(payload) =
1638 std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || drop(value)))
1639 {
1640 task::discard_panic(payload);
1641 }
1642}
1643
1644struct CatchFinished<F> {
1649 inner: Option<std::pin::Pin<Box<F>>>,
1651}
1652
1653impl<F> CatchFinished<F> {
1654 fn new(inner: F) -> Self {
1655 Self {
1656 inner: Some(Box::pin(inner)),
1657 }
1658 }
1659}
1660
1661impl<F: Future<Output = Finished>> Future for CatchFinished<F> {
1662 type Output = Finished;
1663
1664 fn poll(
1665 self: std::pin::Pin<&mut Self>,
1666 cx: &mut std::task::Context<'_>,
1667 ) -> std::task::Poll<Finished> {
1668 let this = self.get_mut();
1669 let Some(inner) = this.inner.as_mut() else {
1670 return std::task::Poll::Pending;
1671 };
1672 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| inner.as_mut().poll(cx))) {
1673 Ok(std::task::Poll::Pending) => std::task::Poll::Pending,
1674 Ok(std::task::Poll::Ready(finished)) => {
1675 this.inner = None;
1676 std::task::Poll::Ready(finished)
1677 }
1678 Err(payload) => {
1679 if let Some(inner) = this.inner.take() {
1682 drop_contained(inner);
1683 }
1684 std::task::Poll::Ready(Finished::Panicked(payload))
1685 }
1686 }
1687 }
1688}
1689
1690pub(crate) struct InvocationEnd {
1693 pub(crate) status: InvocationStatus,
1694 pub(crate) result: Option<String>,
1695 pub(crate) error: Option<WireError>,
1696}
1697
1698impl InvocationEnd {
1699 pub(crate) fn panicked(payload: &(dyn std::any::Any + Send)) -> Self {
1703 Self {
1704 status: InvocationStatus::Retrying,
1705 result: None,
1706 error: Some(WireError::new(
1707 None::<String>,
1708 Some(panic_payload_text(payload)),
1709 )),
1710 }
1711 }
1712
1713 fn retrying(error_type: impl Into<String>, message: String) -> Self {
1716 Self {
1717 status: InvocationStatus::Retrying,
1718 result: None,
1719 error: Some(WireError::new(Some(error_type.into()), Some(message))),
1720 }
1721 }
1722}
1723
1724pub(crate) fn panic_payload_text(payload: &(dyn std::any::Any + Send)) -> String {
1728 if let Some(text) = payload.downcast_ref::<&'static str>() {
1729 (*text).to_owned()
1730 } else if let Some(text) = payload.downcast_ref::<String>() {
1731 text.clone()
1732 } else {
1733 "the invocation panicked with a payload that is not a string".to_owned()
1734 }
1735}
1736
1737async fn end_invocation(
1748 plugins: Option<&plugin::InvocationPlugins>,
1749 log: &engine::CheckpointLog,
1750 ctx: &DurableContext,
1751 end: InvocationEnd,
1752 response: Option<&Result<InvocationResponse, lambda_runtime::Error>>,
1753 fatal_won: bool,
1754) -> Option<Box<dyn std::any::Any + Send + 'static>> {
1755 let plugins = plugins?;
1756 ctx.report_operation_changes().await;
1760 plugins
1761 .invocation_end(
1762 log,
1763 end,
1764 ctx.remaining_time(),
1765 late_panics(response, fatal_won),
1766 )
1767 .await
1768}
1769
1770fn late_panics(
1774 response: Option<&Result<InvocationResponse, lambda_runtime::Error>>,
1775 fatal_won: bool,
1776) -> plugin::LatePanics {
1777 match response {
1778 Some(Ok(_)) if !fatal_won => plugin::LatePanics::Fail,
1779 _ => plugin::LatePanics::Discard,
1780 }
1781}
1782
1783fn create_invocation_plugins(
1790 factories: &plugin::Factories,
1791 lambda_ctx: &lambda_runtime::Context,
1792 execution_arn: &str,
1793 input_payload: Option<&str>,
1794 checkpoint_log: &engine::CheckpointLog,
1795 execution: Option<&plugin::OperationInfo>,
1796 updated_operation_ids: Vec<String>,
1797) -> Option<std::sync::Arc<plugin::InvocationPlugins>> {
1798 plugin::create_plugins(factories, execution.cloned(), || {
1799 plugin::InvocationInfo::new(
1800 lambda_ctx.request_id.clone(),
1801 execution_arn.to_owned(),
1802 input_payload.unwrap_or("{}").to_owned(),
1803 plugin::operations_map(checkpoint_log, execution),
1804 updated_operation_ids,
1805 execution.and_then(plugin::OperationInfo::start_timestamp),
1806 !checkpoint_log.has_records(),
1807 lambda_ctx.xray_trace_id.clone(),
1808 )
1809 })
1810}
1811
1812pub(crate) fn outcome_envelope(
1824 outcome: driver::InvocationOutcome,
1825 ctx: &DurableContext,
1826) -> (
1827 Result<InvocationResponse, lambda_runtime::Error>,
1828 InvocationEnd,
1829) {
1830 match outcome {
1831 driver::InvocationOutcome::Complete(serialized) => (
1832 Ok(InvocationResponse(serde_json::json!({
1833 "Status": "SUCCEEDED",
1834 "Result": serialized
1835 }))),
1836 InvocationEnd {
1837 status: InvocationStatus::Succeeded,
1838 result: Some(serialized),
1839 error: None,
1840 },
1841 ),
1842 driver::InvocationOutcome::Pending => {
1843 let execution_span = ctx.replay_span();
1851 let _execution_scope = execution_span.enter();
1852 tracing_layer::execution_suspended_event(
1853 ctx.execution_arn(),
1854 &ctx.lambda_context().request_id,
1855 );
1856 (
1857 Ok(InvocationResponse(serde_json::json!({
1858 "Status": "PENDING"
1859 }))),
1860 InvocationEnd {
1861 status: InvocationStatus::Pending,
1862 result: None,
1863 error: None,
1864 },
1865 )
1866 }
1867 driver::InvocationOutcome::Fault {
1868 error_type,
1869 message,
1870 } => {
1871 (
1876 Err(lambda_runtime::Error::from(message.clone())),
1877 InvocationEnd::retrying(error_type, message),
1878 )
1879 }
1880 driver::InvocationOutcome::Failed { error } => (
1881 Ok(failed_envelope(&error)),
1882 InvocationEnd {
1883 status: InvocationStatus::Failed,
1884 result: None,
1885 error: Some(error),
1886 },
1887 ),
1888 }
1889}
1890
1891const LAMBDA_RESPONSE_SIZE_LIMIT: usize = 6 * 1024 * 1024 - 50;
1893
1894async fn checkpoint_oversized_result(
1914 ctx: &DurableContext,
1915 serialized: String,
1916) -> (
1917 Result<InvocationResponse, lambda_runtime::Error>,
1918 InvocationEnd,
1919) {
1920 #[expect(clippy::expect_used)]
1921 let epoch_ms = std::time::SystemTime::now()
1924 .duration_since(std::time::UNIX_EPOCH)
1925 .expect("the clock is after the Unix epoch")
1926 .as_millis();
1927 #[expect(clippy::expect_used)]
1928 let update = aws_sdk_lambda::types::OperationUpdate::builder()
1930 .id(engine::compute_wire_id_public(&format!(
1931 "execution-result-{epoch_ms}"
1932 )))
1933 .r#type(aws_sdk_lambda::types::OperationType::Execution)
1934 .action(aws_sdk_lambda::types::OperationAction::Succeed)
1935 .payload(serialized.clone())
1936 .build()
1937 .expect("the update's required fields are set");
1938 match ctx.checkpoint_updates_after_drive(vec![update]).await {
1939 Ok(_) => (
1940 Ok(InvocationResponse(serde_json::json!({
1941 "Status": "SUCCEEDED",
1942 "Result": ""
1943 }))),
1944 InvocationEnd {
1945 status: InvocationStatus::Succeeded,
1946 result: Some(serialized),
1947 error: None,
1948 },
1949 ),
1950 Err(context::WriteError::Failed(e)) if e.is_retryable() => {
1951 let message = error::checkpoint_invocation_failure_message(&e);
1952 (
1953 Err(lambda_runtime::Error::from(message.clone())),
1954 InvocationEnd::retrying(error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE, message),
1955 )
1956 }
1957 Err(context::WriteError::Failed(e)) => {
1958 let error = error::checkpoint_execution_failure_wire(&e);
1959 (
1960 Ok(failed_envelope(&error)),
1961 InvocationEnd {
1962 status: InvocationStatus::Failed,
1963 result: None,
1964 error: Some(error),
1965 },
1966 )
1967 }
1968 Err(context::WriteError::Stopped) => {
1969 outcome_envelope(driver::InvocationOutcome::Pending, ctx)
1970 }
1971 }
1972}
1973
1974fn failed_envelope(error: &WireError) -> InvocationResponse {
1977 let mut error_map = serde_json::Map::new();
1978 error_map.insert(
1979 "ErrorType".to_owned(),
1980 serde_json::Value::String(error.error_type().unwrap_or("Error").to_owned()),
1981 );
1982 error_map.insert(
1983 "ErrorMessage".to_owned(),
1984 serde_json::Value::String(error.error_message().unwrap_or_default().to_owned()),
1985 );
1986 if let Some(data) = error.error_data() {
1987 error_map.insert(
1988 "ErrorData".to_owned(),
1989 serde_json::Value::String(data.to_owned()),
1990 );
1991 }
1992 if !error.stack_trace().is_empty() {
1993 error_map.insert(
1994 "StackTrace".to_owned(),
1995 serde_json::json!(error.stack_trace()),
1996 );
1997 }
1998 InvocationResponse(serde_json::json!({
1999 "Status": "FAILED",
2000 "Error": serde_json::Value::Object(error_map)
2001 }))
2002}
2003
2004pub(crate) fn base_lambda_client_from_options(
2010 sdk_config: Option<SdkConfig>,
2011 lambda_client: Option<aws_sdk_lambda::Client>,
2012) -> Option<aws_sdk_lambda::Client> {
2013 match (lambda_client, sdk_config) {
2014 (Some(client), _) => Some(client),
2015 (None, Some(config)) => Some(aws_sdk_lambda::Client::new(&config)),
2016 (None, None) => None,
2017 }
2018}
2019
2020const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
2023const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(50);
2026const DEFAULT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(55);
2029
2030fn default_lambda_client(config: &SdkConfig) -> aws_sdk_lambda::Client {
2039 use aws_sdk_lambda::config::timeout::TimeoutConfig;
2040 let conf = aws_sdk_lambda::config::Builder::from(config)
2041 .timeout_config(
2042 TimeoutConfig::builder()
2043 .connect_timeout(DEFAULT_CONNECT_TIMEOUT)
2044 .read_timeout(DEFAULT_READ_TIMEOUT)
2045 .operation_attempt_timeout(DEFAULT_ATTEMPT_TIMEOUT)
2046 .build(),
2047 )
2048 .build();
2049 aws_sdk_lambda::Client::from_conf(conf)
2050}
2051
2052pub(crate) struct ClientProvider {
2060 preset: Option<std::sync::Arc<dyn ExecutionClient>>,
2061 default_cell: tokio::sync::OnceCell<std::sync::Arc<dyn ExecutionClient>>,
2062}
2063
2064impl ClientProvider {
2065 pub(crate) fn new(preset: Option<std::sync::Arc<dyn ExecutionClient>>) -> Self {
2068 Self {
2069 preset,
2070 default_cell: tokio::sync::OnceCell::new(),
2071 }
2072 }
2073
2074 pub(crate) async fn get(&self) -> std::sync::Arc<dyn ExecutionClient> {
2077 use std::sync::Arc as StdArc;
2078 if let Some(preset) = &self.preset {
2079 return StdArc::clone(preset);
2080 }
2081 let client = self
2082 .default_cell
2083 .get_or_init(|| async {
2084 let aws_config =
2085 aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
2086 let lambda_client = default_lambda_client(&aws_config);
2087 StdArc::new(LambdaExecutionClient::new(lambda_client))
2088 as StdArc<dyn ExecutionClient>
2089 })
2090 .await;
2091 StdArc::clone(client)
2092 }
2093}
2094
2095#[cfg(test)]
2096mod tests {
2097 use std::future::IntoFuture;
2098
2099 use super::*;
2100
2101 #[test]
2103 fn retry_decision_constructor_builds_retry() {
2104 assert!(matches!(
2105 RetryDecision::retry(Duration::from_secs(3)),
2106 RetryDecision::Retry { delay } if delay == Duration::from_secs(3)
2107 ));
2108 }
2109
2110 #[test]
2111 fn due_at_from_inline_timestamps() {
2112 let payload = serde_json::json!({
2113 "InitialExecutionState": {
2114 "Operations": [
2115 { "Id": "w1", "Type": "WAIT", "Status": "STARTED",
2116 "WaitDetails": { "ScheduledEndTimestamp": "2026-01-01T00:00:00Z" } },
2117 { "Id": "s1", "Type": "STEP", "Status": "PENDING",
2118 "StepDetails": { "NextAttemptTimestamp": "2026-01-01T00:00:05.5Z" } },
2119 { "Id": "w2", "Type": "WAIT", "Status": "STARTED",
2120 "WaitDetails": { "ScheduledEndTimestamp": 1_767_225_600_000_i64 } },
2121 { "Id": "w3", "Type": "WAIT", "Status": "STARTED",
2122 "WaitDetails": { "ScheduledEndTimestamp": "not a date" } },
2123 { "Id": "w4", "Type": "WAIT", "Status": "STARTED",
2124 "WaitDetails": { "ScheduledEndTimestamp": 1.5 } },
2125 { "Id": "w5", "Type": "WAIT", "Status": "STARTED",
2126 "WaitDetails": { "ScheduledEndTimestamp": true } },
2127 ]
2128 }
2129 });
2130 let (log, _) = parse_inline_operations(&payload);
2131 let due = |id: &str| log.get(id).expect("parsed").due_at;
2132 let new_year = std::time::UNIX_EPOCH + Duration::from_hours(490_896);
2133 assert_eq!(due("w1"), Some(new_year));
2134 assert_eq!(due("s1"), Some(new_year + Duration::from_millis(5_500)));
2135 assert_eq!(due("w2"), Some(new_year), "epoch milliseconds");
2136 assert_eq!(due("w3"), None, "a string that does not parse");
2137 assert_eq!(due("w4"), None, "a number that is not an integer");
2138 assert_eq!(due("w5"), None, "another JSON type");
2139 }
2140
2141 struct OutOfRange;
2143
2144 impl TryFrom<OutOfRange> for std::time::SystemTime {
2145 type Error = &'static str;
2146
2147 fn try_from(_: OutOfRange) -> Result<Self, Self::Error> {
2148 Err("outside the range of SystemTime")
2149 }
2150 }
2151
2152 #[test]
2156 fn to_system_time_maps_an_out_of_range_timestamp_to_none() {
2157 let millis = aws_sdk_lambda::primitives::DateTime::from_millis(1_500);
2158 assert_eq!(
2159 to_system_time(millis),
2160 Some(std::time::UNIX_EPOCH + Duration::from_millis(1_500))
2161 );
2162 assert_eq!(to_system_time(OutOfRange), None);
2163 }
2164
2165 #[derive(Debug)]
2166 struct PlainUserError;
2167
2168 impl std::fmt::Display for PlainUserError {
2169 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2170 write!(f, "plain user failure")
2171 }
2172 }
2173
2174 impl std::error::Error for PlainUserError {}
2175
2176 #[test]
2177 fn handler_error_records_no_stack_trace() {
2178 let wire = wire_error_from_box_error(Box::new(PlainUserError));
2179 assert_eq!(wire.error_type(), Some("Error"));
2180 assert_eq!(wire.error_message(), Some("plain user failure"));
2181 assert_eq!(wire.stack_trace(), [] as [String; 0]);
2182 }
2183
2184 struct Refuse;
2186
2187 impl Serialize for Refuse {
2188 fn serialize<S: serde::Serializer>(&self, _serializer: S) -> Result<S::Ok, S::Error> {
2189 Err(<S::Error as serde::ser::Error>::custom("refused"))
2190 }
2191 }
2192
2193 #[test]
2194 fn handler_result_serialization_failure_records_error() {
2195 let Err(wire) = serialize_handler_result(&Refuse) else {
2196 unreachable!("a refusing value must not serialize");
2197 };
2198 assert_eq!(wire.error_type(), Some("Error"));
2199 assert!(
2200 wire.error_message().unwrap_or_default().contains("refused"),
2201 "{:?}",
2202 wire.error_message()
2203 );
2204 assert_eq!(wire.stack_trace(), [] as [String; 0]);
2205 }
2206
2207 #[tokio::test]
2213 async fn tokio_join_accepts_into_future() {
2214 fn check_into_future<T: IntoFuture>(_t: T) {}
2216 let ctx = DurableContext::test_context();
2217 check_into_future(ctx.step(|_| async { Ok(1i32) }));
2218 }
2222
2223 #[test]
2227 fn wrap_returns_callable_service_function() {
2228 fn assert_send_sync<T: Send + Sync>(_t: &T) {}
2229
2230 let service = wrap(
2232 |_event: serde_json::Value, _ctx: DurableContext| async move {
2233 Ok::<String, BoxError>("hello".to_owned())
2234 },
2235 Options::default(),
2236 );
2237
2238 assert_send_sync(&service);
2240
2241 drop(service);
2246 }
2247
2248 fn offline_options() -> Options {
2259 let conf = aws_sdk_lambda::config::Config::builder()
2260 .behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())
2261 .region(aws_sdk_lambda::config::Region::new("us-east-1"))
2262 .build();
2263 Options::builder()
2264 .lambda_client(aws_sdk_lambda::Client::from_conf(conf))
2265 .build()
2266 .expect("offline options build")
2267 }
2268
2269 async fn invoke_wrap_service(
2271 payload: serde_json::Value,
2272 ) -> Result<serde_json::Value, lambda_runtime::Error> {
2273 let service = wrap(
2274 |event: serde_json::Value, _ctx: DurableContext| async move {
2275 Ok::<serde_json::Value, BoxError>(event)
2276 },
2277 offline_options(),
2278 );
2279 let event = lambda_runtime::LambdaEvent::new(
2280 InvocationPayload(payload),
2281 lambda_runtime::Context::default(),
2282 );
2283 service(event).await.map(|response| response.0)
2284 }
2285
2286 #[tokio::test]
2287 async fn wrap_service_missing_arn_fails() {
2288 let payload = serde_json::json!({
2289 "CheckpointToken": "token-abc",
2290 "InitialExecutionState": { "Operations": [] }
2291 });
2292 let err = invoke_wrap_service(payload).await.expect_err("must fail");
2293 assert!(
2294 err.to_string().contains("DurableExecutionArn"),
2295 "error should name the missing field, got: {err}"
2296 );
2297 }
2298
2299 #[tokio::test]
2300 async fn wrap_service_missing_token_fails() {
2301 let payload = serde_json::json!({
2302 "DurableExecutionArn": "arn:aws:lambda:us-east-1:123456789012:function:test",
2303 "InitialExecutionState": { "Operations": [] }
2304 });
2305 let err = invoke_wrap_service(payload).await.expect_err("must fail");
2306 assert!(
2307 err.to_string().contains("CheckpointToken"),
2308 "error should name the missing field, got: {err}"
2309 );
2310 }
2311
2312 #[tokio::test]
2313 async fn wrap_service_envelope_free_payload_fails() {
2314 let payload = serde_json::json!({ "count": 42 });
2318 let err = invoke_wrap_service(payload).await.expect_err("must fail");
2319 assert!(
2320 err.to_string().contains("not a durable execution envelope"),
2321 "error should describe the missing envelope, got: {err}"
2322 );
2323 }
2324
2325 #[tokio::test]
2326 async fn wrap_service_valid_envelope_succeeds() {
2327 let payload = serde_json::json!({
2328 "DurableExecutionArn": "arn:aws:lambda:us-east-1:123456789012:function:test",
2329 "CheckpointToken": "token-abc",
2330 "InitialExecutionState": {
2331 "Operations": [{
2332 "Id": "root",
2333 "Type": "Execution",
2334 "Status": "STARTED",
2335 "ExecutionDetails": { "InputPayload": "{\"count\":42}" }
2336 }]
2337 }
2338 });
2339 let response = invoke_wrap_service(payload).await.expect("must succeed");
2340 assert_eq!(
2341 response.get("Status").and_then(serde_json::Value::as_str),
2342 Some("SUCCEEDED"),
2343 "unexpected response: {response}"
2344 );
2345 let result_json = response
2346 .get("Result")
2347 .and_then(serde_json::Value::as_str)
2348 .expect("Result should be a serialized JSON string");
2349 let echoed: serde_json::Value = serde_json::from_str(result_json).unwrap();
2350 assert_eq!(echoed, serde_json::json!({ "count": 42 }));
2351 }
2352
2353 #[tokio::test]
2357 #[expect(clippy::disallowed_methods)]
2360 async fn handler_panic_leaves_sdk_with_original_payload() {
2361 #[expect(clippy::panic)] fn boom() -> Result<(), BoxError> {
2363 panic!("root boom")
2364 }
2365 let client: std::sync::Arc<dyn ExecutionClient> =
2366 std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
2367 let service = wrap_with_provider(
2368 |_event: serde_json::Value, _ctx: DurableContext| async move { boom() },
2369 ClientProvider::new(Some(client)),
2370 Duration::ZERO,
2371 0,
2372 std::sync::Arc::from(Vec::new()),
2373 );
2374 let payload = serde_json::json!({
2375 "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
2376 "CheckpointToken": "token-abc",
2377 "InitialExecutionState": {
2378 "Operations": [{
2379 "Id": "root",
2380 "Type": "Execution",
2381 "Status": "STARTED",
2382 "ExecutionDetails": { "InputPayload": "{}" }
2383 }]
2384 }
2385 });
2386 let event = lambda_runtime::LambdaEvent::new(
2387 InvocationPayload(payload),
2388 lambda_runtime::Context::default(),
2389 );
2390 let join_err = tokio::spawn(service(event))
2391 .await
2392 .expect_err("the handler panic leaves the service, and no envelope is produced");
2393 assert!(join_err.is_panic(), "a panic, not a cancellation");
2394 let payload = join_err.into_panic();
2395 assert_eq!(payload.downcast_ref::<&str>(), Some(&"root boom"));
2396 }
2397
2398 fn paginated_envelope() -> serde_json::Value {
2403 serde_json::json!({
2404 "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
2405 "CheckpointToken": "token-abc",
2406 "InitialExecutionState": {
2407 "Operations": [{
2408 "Id": "root",
2409 "Type": "Execution",
2410 "Status": "STARTED",
2411 "ExecutionDetails": { "InputPayload": "{}" }
2412 }],
2413 "NextMarker": "page-2"
2414 }
2415 })
2416 }
2417
2418 async fn invoke_paginated(
2422 sdk_client: aws_sdk_lambda::Client,
2423 ) -> (Result<serde_json::Value, lambda_runtime::Error>, bool) {
2424 invoke_paginated_with(
2425 Options::builder()
2426 .lambda_client(sdk_client)
2427 .build()
2428 .expect("options build"),
2429 )
2430 .await
2431 }
2432
2433 async fn invoke_paginated_with(
2437 options: Options,
2438 ) -> (Result<serde_json::Value, lambda_runtime::Error>, bool) {
2439 use std::sync::atomic::{AtomicBool, Ordering};
2440
2441 let ran = std::sync::Arc::new(AtomicBool::new(false));
2442 let handler_ran = std::sync::Arc::clone(&ran);
2443 let service = wrap(
2444 move |_event: serde_json::Value, _ctx: DurableContext| {
2445 handler_ran.store(true, Ordering::SeqCst);
2446 async move { Ok::<(), BoxError>(()) }
2447 },
2448 options,
2449 );
2450 let event = lambda_runtime::LambdaEvent::new(
2451 InvocationPayload(paginated_envelope()),
2452 lambda_runtime::Context::default(),
2453 );
2454 let result = service(event).await.map(|response| response.0);
2455 (result, ran.load(Ordering::SeqCst))
2456 }
2457
2458 #[tokio::test]
2461 async fn bootstrap_kms_error_returns_failed_envelope() {
2462 use aws_smithy_mocks::{mock, mock_client};
2463
2464 let rule =
2465 mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_http_response(|| {
2466 client::mock_aws_error_response(400, "KMSAccessDeniedException", "kms denied")
2467 });
2468 let (result, ran) = invoke_paginated(mock_client!(aws_sdk_lambda, [&rule])).await;
2469
2470 let response = result.expect("a KMS error returns an envelope");
2471 assert_eq!(
2472 response.get("Status").and_then(serde_json::Value::as_str),
2473 Some("FAILED"),
2474 "unexpected response: {response}"
2475 );
2476 let error = response
2477 .get("Error")
2478 .expect("a FAILED envelope has an Error");
2479 assert_eq!(
2480 error.get("ErrorType").and_then(serde_json::Value::as_str),
2481 Some("KMSAccessDeniedException")
2482 );
2483 assert_eq!(
2484 error
2485 .get("ErrorMessage")
2486 .and_then(serde_json::Value::as_str),
2487 Some("kms denied")
2488 );
2489 assert!(!ran, "the handler must not run");
2490 }
2491
2492 #[tokio::test]
2496 async fn bootstrap_kms_error_records_aws_error_code() {
2497 use aws_smithy_mocks::{mock, mock_client};
2498
2499 let rule =
2500 mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_http_response(|| {
2501 client::mock_aws_error_response(400, "KMSAccessDeniedException", "kms denied")
2502 });
2503 let client = mock_client!(aws_sdk_lambda, [&rule]);
2504 let (result, ran) = invoke_paginated_with(
2505 Options::builder()
2506 .execution_client(std::sync::Arc::new(LambdaExecutionClient::new(client)))
2507 .build()
2508 .expect("options build"),
2509 )
2510 .await;
2511
2512 let response = result.expect("a KMS error returns an envelope");
2513 assert_eq!(
2514 response.get("Status").and_then(serde_json::Value::as_str),
2515 Some("FAILED"),
2516 "unexpected response: {response}"
2517 );
2518 let error = response
2519 .get("Error")
2520 .expect("a FAILED envelope has an Error");
2521 assert_eq!(
2522 error.get("ErrorType").and_then(serde_json::Value::as_str),
2523 Some("KMSAccessDeniedException")
2524 );
2525 assert_eq!(
2526 error
2527 .get("ErrorMessage")
2528 .and_then(serde_json::Value::as_str),
2529 Some("kms denied")
2530 );
2531 assert!(!ran, "the handler must not run");
2532 }
2533
2534 #[tokio::test]
2538 async fn lambda_checkpoint_rejection_records_the_service_message() {
2539 use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionOutput;
2540 use aws_smithy_mocks::{RuleMode, mock, mock_client};
2541
2542 let rejected = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
2543 .then_http_response(|| {
2544 client::mock_aws_error_response(
2545 400,
2546 "InvalidParameterValueException",
2547 "bad payload",
2548 )
2549 });
2550 let accepted =
2551 mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_output(|| {
2552 CheckpointDurableExecutionOutput::builder()
2553 .checkpoint_token("tok-2")
2554 .build()
2555 });
2556 let client = mock_client!(aws_sdk_lambda, RuleMode::Sequential, [&rejected, &accepted]);
2557 let service = wrap(
2558 |_event: serde_json::Value, ctx: DurableContext| async move {
2559 let value: u32 = ctx.step(|_| async { Ok(7_u32) }).name("one").await?;
2560 Ok::<u32, BoxError>(value)
2561 },
2562 Options::builder()
2563 .execution_client(std::sync::Arc::new(LambdaExecutionClient::new(client)))
2564 .build()
2565 .expect("options build"),
2566 );
2567 let event = lambda_runtime::LambdaEvent::new(
2568 InvocationPayload(serde_json::json!({
2569 "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
2570 "CheckpointToken": "token-abc",
2571 "InitialExecutionState": {
2572 "Operations": [{
2573 "Id": "root",
2574 "Type": "Execution",
2575 "Status": "STARTED",
2576 "ExecutionDetails": { "InputPayload": "{}" }
2577 }]
2578 }
2579 })),
2580 lambda_runtime::Context::default(),
2581 );
2582 let response = service(event)
2583 .await
2584 .map(|response| response.0)
2585 .expect("a rejected checkpoint returns an envelope");
2586
2587 assert_eq!(
2588 response.get("Status").and_then(serde_json::Value::as_str),
2589 Some("FAILED"),
2590 "unexpected response: {response}"
2591 );
2592 let error = response
2593 .get("Error")
2594 .expect("a FAILED envelope has an Error");
2595 assert_eq!(
2596 error.get("ErrorType").and_then(serde_json::Value::as_str),
2597 Some("CheckpointUnrecoverableExecutionError")
2598 );
2599 assert_eq!(
2600 error
2601 .get("ErrorMessage")
2602 .and_then(serde_json::Value::as_str),
2603 Some("[Unrecoverable Execution] Checkpoint failed: bad payload")
2604 );
2605 }
2606
2607 #[tokio::test]
2610 async fn bootstrap_retryable_error_fails_the_invocation() {
2611 use aws_smithy_mocks::{mock, mock_client};
2612
2613 for status in [503, 403] {
2614 let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
2615 .sequence()
2616 .http_status(status, None)
2617 .repeatedly()
2618 .build();
2619 let (result, ran) = invoke_paginated(mock_client!(aws_sdk_lambda, [&rule])).await;
2620
2621 let err = result.expect_err("a retryable error fails the invocation");
2622 assert!(
2623 err.to_string().contains("failed to paginate initial state"),
2624 "status {status}: {err}"
2625 );
2626 assert!(!ran, "the handler must not run");
2627 }
2628 }
2629
2630 fn step_op_with_status(status: &str) -> serde_json::Value {
2634 serde_json::json!({
2635 "Id": "op-1",
2636 "Type": "Step",
2637 "Status": status,
2638 "StepDetails": {}
2639 })
2640 }
2641
2642 #[test]
2643 fn parse_status_timed_out_wire_value_maps_to_timed_out() {
2644 let (_, record) = parse_single_operation(&step_op_with_status("TIMED_OUT")).unwrap();
2645 assert_eq!(record.status, engine::CheckpointStatus::TimedOut);
2646 }
2647
2648 #[test]
2652 fn parse_status_timedout_without_underscore_is_unrecognized() {
2653 let (_, record) = parse_single_operation(&step_op_with_status("TIMEDOUT")).unwrap();
2654 assert_eq!(
2655 record.status,
2656 engine::CheckpointStatus::Unknown("TIMEDOUT".to_owned())
2657 );
2658 }
2659
2660 #[test]
2663 fn parse_unknown_status_carries_raw_value() {
2664 let (_, record) = parse_single_operation(&step_op_with_status("Paused")).unwrap();
2665 assert_eq!(
2666 record.status,
2667 engine::CheckpointStatus::Unknown("Paused".to_owned())
2668 );
2669 }
2670
2671 #[test]
2674 fn parse_known_statuses_map_explicitly() {
2675 let cases = [
2676 ("STARTED", engine::CheckpointStatus::Started),
2677 ("PENDING", engine::CheckpointStatus::Pending),
2678 ("READY", engine::CheckpointStatus::Ready),
2679 ("SUCCEEDED", engine::CheckpointStatus::Succeeded),
2680 ("FAILED", engine::CheckpointStatus::Failed),
2681 ("CANCELLED", engine::CheckpointStatus::Cancelled),
2682 ("TIMED_OUT", engine::CheckpointStatus::TimedOut),
2683 ("STOPPED", engine::CheckpointStatus::Stopped),
2684 ("Succeeded", engine::CheckpointStatus::Succeeded),
2685 ];
2686 for (wire, expected) in cases {
2687 let (_, record) = parse_single_operation(&step_op_with_status(wire)).unwrap();
2688 assert_eq!(record.status, expected, "wire status {wire:?}");
2689 }
2690 }
2691
2692 #[test]
2695 fn parse_callback_details_extracts_result() {
2696 let op = serde_json::json!({
2697 "Id": "abc123",
2698 "Type": "Callback",
2699 "Status": "SUCCEEDED",
2700 "CallbackDetails": {
2701 "CallbackId": "cb-42",
2702 "Result": "\"hello from callback\""
2703 }
2704 });
2705
2706 let parsed = parse_single_operation(&op);
2707 assert!(parsed.is_some());
2708 let (id, record) = parsed.unwrap();
2709 assert_eq!(id, "abc123");
2710 assert_eq!(record.status, engine::CheckpointStatus::Succeeded);
2711 assert_eq!(record.callback_id.as_deref(), Some("cb-42"));
2712 assert_eq!(record.result.as_deref(), Some("\"hello from callback\""));
2713 }
2714
2715 #[test]
2716 fn parse_callback_details_extracts_error() {
2717 let op = serde_json::json!({
2718 "Id": "abc456",
2719 "Type": "Callback",
2720 "Status": "FAILED",
2721 "CallbackDetails": {
2722 "CallbackId": "cb-99",
2723 "Error": {
2724 "ErrorType": "NotApproved",
2725 "ErrorMessage": "request was denied"
2726 }
2727 }
2728 });
2729
2730 let parsed = parse_single_operation(&op);
2731 assert!(parsed.is_some());
2732 let (id, record) = parsed.unwrap();
2733 assert_eq!(id, "abc456");
2734 assert_eq!(record.status, engine::CheckpointStatus::Failed);
2735 assert_eq!(record.callback_id.as_deref(), Some("cb-99"));
2736 assert_eq!(record.error_type.as_deref(), Some("NotApproved"));
2737 assert_eq!(record.error_message.as_deref(), Some("request was denied"));
2738 }
2739
2740 #[test]
2741 fn parse_callback_details_result_does_not_override_step_result() {
2742 let op = serde_json::json!({
2745 "Id": "abc789",
2746 "Type": "Callback",
2747 "Status": "SUCCEEDED",
2748 "StepDetails": {
2749 "Result": "\"from step\""
2750 },
2751 "CallbackDetails": {
2752 "CallbackId": "cb-1",
2753 "Result": "\"from callback\""
2754 }
2755 });
2756
2757 let parsed = parse_single_operation(&op);
2758 assert!(parsed.is_some());
2759 let (_, record) = parsed.unwrap();
2760 assert_eq!(record.result.as_deref(), Some("\"from step\""));
2761 }
2762
2763 #[test]
2764 fn parse_inline_operations_handles_callback_succeeded() {
2765 let payload = serde_json::json!({
2766 "InitialExecutionState": {
2767 "Operations": [
2768 {
2769 "Id": "exec-0",
2770 "Type": "Execution",
2771 "Status": "STARTED"
2772 },
2773 {
2774 "Id": "wire-id-1",
2775 "Type": "Callback",
2776 "Status": "SUCCEEDED",
2777 "CallbackDetails": {
2778 "CallbackId": "cb-id-123",
2779 "Result": "\"payload\""
2780 }
2781 }
2782 ]
2783 }
2784 });
2785
2786 let (log, marker) = parse_inline_operations(&payload);
2787 assert!(marker.is_none());
2788 let record = log.get("wire-id-1");
2789 assert!(record.is_some());
2790 let record = record.unwrap();
2791 assert_eq!(record.callback_id.as_deref(), Some("cb-id-123"));
2792 assert_eq!(record.result.as_deref(), Some("\"payload\""));
2793 }
2794
2795 #[test]
2799 fn parse_inline_operations_returns_next_marker() {
2800 let payload = serde_json::json!({
2801 "InitialExecutionState": {
2802 "Operations": [
2803 {
2804 "Id": "exec-0",
2805 "Type": "Execution",
2806 "Status": "STARTED"
2807 },
2808 {
2809 "Id": "wire-id-1",
2810 "Type": "Step",
2811 "Status": "SUCCEEDED",
2812 "StepDetails": {
2813 "Attempt": 1,
2814 "Result": "\"hello\""
2815 }
2816 }
2817 ],
2818 "NextMarker": "page-token-2"
2819 }
2820 });
2821
2822 let (log, marker) = parse_inline_operations(&payload);
2823 let record = log.get("wire-id-1");
2825 assert!(record.is_some());
2826 assert_eq!(marker, Some("page-token-2".to_owned()));
2828 }
2829
2830 #[test]
2832 fn parse_inline_operations_ignores_empty_marker() {
2833 let payload = serde_json::json!({
2834 "InitialExecutionState": {
2835 "Operations": [
2836 {
2837 "Id": "exec-0",
2838 "Type": "Execution",
2839 "Status": "STARTED"
2840 }
2841 ],
2842 "NextMarker": ""
2843 }
2844 });
2845
2846 let (_log, marker) = parse_inline_operations(&payload);
2847 assert_eq!(marker, None);
2848 }
2849
2850 #[test]
2855 fn parse_inline_operations_keeps_marker_without_operations() {
2856 let payload = serde_json::json!({
2857 "InitialExecutionState": {
2858 "NextMarker": "page-token-1"
2859 }
2860 });
2861
2862 let (log, marker) = parse_inline_operations(&payload);
2863 assert!(!log.has_records());
2865 assert_eq!(marker, Some("page-token-1".to_owned()));
2867 }
2868
2869 fn make_test_step_op(id: &str, result: &str) -> aws_sdk_lambda::types::Operation {
2871 aws_sdk_lambda::types::Operation::builder()
2872 .id(id)
2873 .r#type(aws_sdk_lambda::types::OperationType::Step)
2874 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2875 .start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
2876 .step_details(
2877 aws_sdk_lambda::types::StepDetails::builder()
2878 .result(result)
2879 .build(),
2880 )
2881 .build()
2882 .unwrap()
2883 }
2884
2885 fn make_test_execution_op(input: &str) -> aws_sdk_lambda::types::Operation {
2887 aws_sdk_lambda::types::Operation::builder()
2888 .id("execution")
2889 .r#type(aws_sdk_lambda::types::OperationType::Execution)
2890 .status(aws_sdk_lambda::types::OperationStatus::Started)
2891 .start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
2892 .execution_details(
2893 aws_sdk_lambda::types::ExecutionDetails::builder()
2894 .input_payload(input)
2895 .build(),
2896 )
2897 .build()
2898 .unwrap()
2899 }
2900
2901 #[tokio::test]
2905 async fn resolve_bootstrap_log_paginates_when_marker_present() {
2906 let all_ops = vec![
2907 make_test_execution_op("{\"fetched\":true}"),
2908 make_test_step_op("step-1", "\"r1\""),
2909 make_test_step_op("step-2", "\"r2\""),
2910 ];
2911 let client = client::InMemoryExecutionClient::new(all_ops);
2912
2913 let inline_log = engine::CheckpointLog::empty();
2915
2916 let result = client::resolve_bootstrap_log(
2917 &client,
2918 "arn:test",
2919 "token",
2920 client::BootstrapState {
2921 log: inline_log,
2922 input_payload: None,
2923 execution: None,
2924 },
2925 Some("page-2-marker"),
2926 )
2927 .await;
2928
2929 assert!(result.is_ok());
2930 let client::BootstrapState {
2931 log,
2932 input_payload: input,
2933 execution,
2934 } = result.unwrap();
2935 let execution = execution.expect("the fetched Execution operation");
2936 assert_eq!(execution.id(), "execution");
2937 assert_eq!(execution.kind(), Some(OperationKind::Execution));
2938 assert_eq!(execution.status(), Some(OperationStatus::Started));
2939 assert_eq!(
2940 execution.start_timestamp(),
2941 Some(std::time::UNIX_EPOCH),
2942 "the fetched start timestamp"
2943 );
2944 assert_eq!(
2945 input.as_deref(),
2946 Some("{\"fetched\":true}"),
2947 "the input must come from the fetched Execution operation"
2948 );
2949 assert!(
2950 log.get("execution").is_none(),
2951 "the Execution operation must not enter the log"
2952 );
2953 assert!(log.get("step-1").is_some(), "step-1 must be in the log");
2955 assert!(
2956 log.get("step-2").is_some(),
2957 "step-2 must be in the log (from page 2)"
2958 );
2959
2960 let count = *client.get_state_call_count.lock().unwrap();
2962 assert_eq!(
2963 count, 1,
2964 "get_state must be called exactly once when marker is present"
2965 );
2966 }
2967
2968 #[tokio::test]
2971 async fn resolve_bootstrap_log_skips_pagination_when_no_marker() {
2972 let client = client::InMemoryExecutionClient::new(Vec::new());
2973
2974 let inline_log = engine::CheckpointLog::empty();
2975 inline_log.insert(
2977 "existing-op".to_owned(),
2978 engine::CheckpointRecord {
2979 id: "existing-op".to_owned(),
2980 status: engine::CheckpointStatus::Succeeded,
2981 result: Some("\"inline\"".to_owned()),
2982 error_type: None,
2983 error_message: None,
2984 error_data: None,
2985 stack_trace: None,
2986 attempt: 1,
2987 invoke_result: None,
2988 invoke_error_type: None,
2989 invoke_error_message: None,
2990 invoke_error_data: None,
2991 invoke_stack_trace: None,
2992 replay_children: false,
2993 callback_id: None,
2994 op_type: None,
2995 sub_type: None,
2996 due_at: None,
2997 op_name: None,
2998 parent_id: None,
2999 start_timestamp: None,
3000 end_timestamp: None,
3001 },
3002 );
3003
3004 let inline_execution = execution_info_from_json(&serde_json::json!({
3005 "Id": "inline-exec",
3006 "Type": "EXECUTION",
3007 "Status": "STARTED",
3008 "StartTimestamp": 4000
3009 }));
3010 assert!(inline_execution.is_some());
3011 let result = client::resolve_bootstrap_log(
3012 &client,
3013 "arn:test",
3014 "token",
3015 client::BootstrapState {
3016 log: inline_log,
3017 input_payload: Some("\"inline-input\"".to_owned()),
3018 execution: inline_execution,
3019 },
3020 None,
3021 )
3022 .await;
3023
3024 assert!(result.is_ok());
3025 let client::BootstrapState {
3026 log,
3027 input_payload: input,
3028 execution,
3029 } = result.unwrap();
3030 let execution = execution.expect("the inline Execution operation");
3032 assert_eq!(execution.id(), "inline-exec");
3033 assert_eq!(
3034 execution.start_timestamp(),
3035 Some(std::time::UNIX_EPOCH + Duration::from_secs(4))
3036 );
3037 assert_eq!(input.as_deref(), Some("\"inline-input\""));
3039 assert!(
3041 log.get("existing-op").is_some(),
3042 "inline op must be preserved"
3043 );
3044
3045 let count = *client.get_state_call_count.lock().unwrap();
3047 assert_eq!(
3048 count, 0,
3049 "get_state must not be called when no marker is present"
3050 );
3051 }
3052
3053 #[test]
3058 fn sdk_config_measurably_alters_client_construction() {
3059 let sdk_config = SdkConfig::builder()
3060 .behavior_version(aws_config::BehaviorVersion::latest())
3061 .region(aws_sdk_lambda::config::Region::new("eu-west-1"))
3062 .build();
3063 let client = base_lambda_client_from_options(Some(sdk_config), None)
3064 .expect("sdk_config must yield a client");
3065 assert_eq!(
3066 client.config().region().map(ToString::to_string),
3067 Some("eu-west-1".to_owned()),
3068 "the supplied sdk_config's region must flow into the built client"
3069 );
3070 }
3071
3072 #[test]
3075 fn supplied_lambda_client_is_the_one_used() {
3076 let conf = aws_sdk_lambda::config::Config::builder()
3077 .behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())
3078 .region(aws_sdk_lambda::config::Region::new("ap-south-1"))
3079 .build();
3080 let supplied = aws_sdk_lambda::Client::from_conf(conf);
3081 let resolved = base_lambda_client_from_options(None, Some(supplied))
3082 .expect("lambda_client must be returned");
3083 assert_eq!(
3084 resolved.config().region().map(ToString::to_string),
3085 Some("ap-south-1".to_owned()),
3086 "the supplied lambda_client must be used verbatim, not replaced"
3087 );
3088 }
3089
3090 #[test]
3093 fn no_options_defers_client_to_ambient_default() {
3094 assert!(base_lambda_client_from_options(None, None).is_none());
3095 }
3096
3097 #[tokio::test]
3101 async fn client_provider_reuses_preset_across_invocations() {
3102 use crate::client::InMemoryExecutionClient;
3103 use std::sync::Arc as StdArc;
3104
3105 let preset: StdArc<dyn ExecutionClient> =
3106 StdArc::new(InMemoryExecutionClient::new(Vec::new()));
3107 let provider = ClientProvider::new(Some(StdArc::clone(&preset)));
3108
3109 let first = provider.get().await;
3110 let second = provider.get().await;
3111 assert!(
3112 StdArc::ptr_eq(&first, &second),
3113 "the client must be reused, not rebuilt, across invocations"
3114 );
3115 assert!(
3116 StdArc::ptr_eq(&first, &preset),
3117 "the reused client must be exactly the one supplied via Options"
3118 );
3119 }
3120
3121 #[test]
3124 fn parse_envelope_valid_payload() {
3125 let payload = serde_json::json!({
3126 "DurableExecutionArn": "arn:aws:lambda:us-west-2:123456789012:function:test",
3127 "CheckpointToken": "tok-abc",
3128 "InitialExecutionState": { "Operations": [] }
3129 });
3130 let result = parse_envelope(&payload);
3131 assert!(result.is_ok());
3132 let envelope = result.unwrap().expect("envelope should be Some");
3133 assert_eq!(
3134 envelope.execution_arn,
3135 "arn:aws:lambda:us-west-2:123456789012:function:test"
3136 );
3137 assert_eq!(envelope.checkpoint_token, "tok-abc");
3138 }
3139
3140 #[test]
3141 fn parse_envelope_missing_arn_errors() {
3142 let payload = serde_json::json!({
3143 "CheckpointToken": "tok-abc",
3144 "InitialExecutionState": { "Operations": [] }
3145 });
3146 let err = parse_envelope(&payload).unwrap_err();
3147 let msg = err.to_string();
3148 assert!(
3149 msg.contains("DurableExecutionArn") && msg.contains("missing"),
3150 "error should name the missing field, got: {msg}"
3151 );
3152 }
3153
3154 #[test]
3155 fn parse_envelope_missing_token_errors() {
3156 let payload = serde_json::json!({
3157 "DurableExecutionArn": "arn:test",
3158 "InitialExecutionState": { "Operations": [] }
3159 });
3160 let err = parse_envelope(&payload).unwrap_err();
3161 let msg = err.to_string();
3162 assert!(
3163 msg.contains("CheckpointToken") && msg.contains("missing"),
3164 "error should name the missing field, got: {msg}"
3165 );
3166 }
3167
3168 #[test]
3169 fn parse_envelope_arn_wrong_type_errors() {
3170 let payload = serde_json::json!({
3171 "DurableExecutionArn": 12345,
3172 "CheckpointToken": "tok-abc",
3173 "InitialExecutionState": { "Operations": [] }
3174 });
3175 let err = parse_envelope(&payload).unwrap_err();
3176 let msg = err.to_string();
3177 assert!(
3178 msg.contains("DurableExecutionArn") && msg.contains("not a string"),
3179 "error should note the type mismatch, got: {msg}"
3180 );
3181 }
3182
3183 #[test]
3184 fn parse_envelope_token_wrong_type_errors() {
3185 let payload = serde_json::json!({
3186 "DurableExecutionArn": "arn:test",
3187 "CheckpointToken": ["not", "a", "string"],
3188 "InitialExecutionState": { "Operations": [] }
3189 });
3190 let err = parse_envelope(&payload).unwrap_err();
3191 let msg = err.to_string();
3192 assert!(
3193 msg.contains("CheckpointToken") && msg.contains("not a string"),
3194 "error should note the type mismatch, got: {msg}"
3195 );
3196 }
3197
3198 #[test]
3199 fn envelope_updated_operation_ids_are_read() {
3200 let ids = |extra: serde_json::Value| {
3201 let mut payload = serde_json::json!({
3202 "DurableExecutionArn": "arn:test",
3203 "CheckpointToken": "tok",
3204 });
3205 if let (Some(map), serde_json::Value::Object(extra)) = (payload.as_object_mut(), extra)
3206 {
3207 map.extend(extra);
3208 }
3209 parse_envelope(&payload)
3210 .unwrap()
3211 .unwrap()
3212 .updated_operation_ids
3213 };
3214 assert_eq!(
3215 ids(serde_json::json!({ "UpdatedOperationIds": ["a", 7, "b"] })),
3216 ["a", "b"]
3217 );
3218 assert_eq!(ids(serde_json::json!({})), Vec::<String>::new());
3219 assert_eq!(
3220 ids(serde_json::json!({ "UpdatedOperationIds": "a" })),
3221 Vec::<String>::new()
3222 );
3223 }
3224
3225 #[test]
3226 fn inline_operation_parent_and_timestamps() {
3227 let op = serde_json::json!({
3228 "Id": "s",
3229 "Type": "STEP",
3230 "Status": "STARTED",
3231 "ParentId": "p",
3232 "StartTimestamp": 1500,
3233 "EndTimestamp": "later"
3234 });
3235 let (_, record) = parse_single_operation(&op).unwrap();
3236 assert_eq!(record.parent_id.as_deref(), Some("p"));
3237 assert_eq!(
3238 record.start_timestamp,
3239 Some(std::time::UNIX_EPOCH + Duration::from_millis(1500))
3240 );
3241 assert_eq!(record.end_timestamp, None);
3242 }
3243
3244 #[test]
3247 fn unknown_type_and_status_keep_wire_tokens() {
3248 let op = serde_json::json!({
3249 "Id": "f",
3250 "Type": "FUTURE_OP",
3251 "Status": "FUTURE_STATUS"
3252 });
3253 let (id, record) = parse_single_operation(&op).unwrap();
3254 let info = plugin::OperationInfo::from_record(&id, &record);
3255 assert_eq!(info.kind(), None);
3256 assert_eq!(info.status(), None);
3257 assert_eq!(info.type_str(), Some("FUTURE_OP"));
3258 assert_eq!(info.status_str(), Some("FUTURE_STATUS"));
3259 }
3260
3261 #[tokio::test]
3265 async fn bootstrap_state_reads_execution() {
3266 let inline = execution_info_from_json(&serde_json::json!({
3267 "Id": "exec",
3268 "Type": "EXECUTION",
3269 "Status": "STARTED",
3270 "StartTimestamp": 2000
3271 }))
3272 .expect("an inline EXECUTION operation");
3273 assert_eq!(
3274 inline.start_timestamp(),
3275 Some(std::time::UNIX_EPOCH + Duration::from_secs(2))
3276 );
3277
3278 let fetched_execution = aws_sdk_lambda::types::Operation::builder()
3279 .id("exec")
3280 .r#type(aws_sdk_lambda::types::OperationType::Execution)
3281 .status(aws_sdk_lambda::types::OperationStatus::Started)
3282 .start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(3))
3283 .build()
3284 .unwrap();
3285 let client = client::InMemoryExecutionClient::new(vec![
3286 fetched_execution,
3287 make_test_step_op("s1", "\"r\""),
3288 ]);
3289 let state = client::resolve_bootstrap_log(
3290 &client,
3291 "arn:test",
3292 "token",
3293 client::BootstrapState {
3294 log: engine::CheckpointLog::empty(),
3295 input_payload: None,
3296 execution: None,
3297 },
3298 Some("m"),
3299 )
3300 .await
3301 .unwrap();
3302 assert_eq!(
3303 state.execution.and_then(|e| e.start_timestamp()),
3304 Some(std::time::UNIX_EPOCH + Duration::from_secs(3))
3305 );
3306 }
3307
3308 #[test]
3312 fn execution_info_from_json_rejects_other_operations() {
3313 assert!(
3314 execution_info_from_json(&serde_json::json!({ "Id": "s", "Type": "STEP" })).is_none()
3315 );
3316 assert!(execution_info_from_json(&serde_json::json!({ "Type": "EXECUTION" })).is_none());
3317 assert!(execution_info_from_json(&serde_json::json!({ "Id": "e" })).is_none());
3318 let info = execution_info_from_json(&serde_json::json!({
3319 "Id": "e",
3320 "Type": "Execution",
3321 "ParentId": "p",
3322 "EndTimestamp": 5000
3323 }))
3324 .expect("the PascalCase spelling is an EXECUTION operation");
3325 assert_eq!(info.status(), Some(OperationStatus::Started));
3326 assert_eq!(info.parent_id(), Some("p"));
3327 assert_eq!(
3328 info.end_timestamp(),
3329 Some(std::time::UNIX_EPOCH + Duration::from_secs(5))
3330 );
3331 }
3332
3333 #[derive(Clone, Default)]
3336 struct RecordingFactory(std::sync::Arc<std::sync::Mutex<Vec<plugin::InvocationInfo>>>);
3337
3338 impl plugin::PluginFactory for RecordingFactory {
3339 fn create_plugin(
3340 &self,
3341 info: &plugin::InvocationInfo,
3342 ) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
3343 self.0.lock().unwrap().push(info.clone());
3344 Ok(None)
3345 }
3346 }
3347
3348 async fn infos_for(
3351 payload: serde_json::Value,
3352 lambda_ctx: lambda_runtime::Context,
3353 ) -> Vec<plugin::InvocationInfo> {
3354 let factory = RecordingFactory::default();
3355 let options = Options::builder()
3356 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
3357 Vec::new(),
3358 )))
3359 .plugin(factory.clone())
3360 .build()
3361 .unwrap();
3362 let service = wrap(
3363 |_event: serde_json::Value, _ctx: DurableContext| async move {
3364 Ok::<serde_json::Value, BoxError>(serde_json::Value::Null)
3365 },
3366 options,
3367 );
3368 service(lambda_runtime::LambdaEvent::new(
3369 InvocationPayload(payload),
3370 lambda_ctx,
3371 ))
3372 .await
3373 .unwrap();
3374 factory.0.lock().unwrap().clone()
3375 }
3376
3377 fn envelope_with(operations: &serde_json::Value) -> serde_json::Value {
3378 serde_json::json!({
3379 "DurableExecutionArn": "arn:test",
3380 "CheckpointToken": "tok",
3381 "InitialExecutionState": { "Operations": operations }
3382 })
3383 }
3384
3385 #[tokio::test]
3386 async fn operations_map_contains_execution() {
3387 let payload = envelope_with(&serde_json::json!([
3388 {
3389 "Id": "exec-1",
3390 "Type": "EXECUTION",
3391 "Status": "STARTED",
3392 "StartTimestamp": 7000,
3393 "ExecutionDetails": { "InputPayload": "{}" }
3394 },
3395 { "Id": "s1", "Type": "STEP", "Status": "SUCCEEDED", "ParentId": "exec-1" },
3396 { "Id": "c1", "Type": "CONTEXT", "Status": "STARTED" }
3397 ]));
3398 let infos = infos_for(payload, lambda_runtime::Context::default()).await;
3399 assert_eq!(infos.len(), 1);
3400 let operations = infos.first().unwrap().operations();
3401 let exec = operations.get("exec-1").expect("the EXECUTION operation");
3402 assert_eq!(exec.kind(), Some(OperationKind::Execution));
3403 assert_eq!(exec.status(), Some(OperationStatus::Started));
3404 assert_eq!(exec.status_str(), Some("STARTED"));
3405 assert_eq!(
3406 exec.start_timestamp(),
3407 Some(std::time::UNIX_EPOCH + Duration::from_secs(7))
3408 );
3409 assert_eq!(
3410 operations
3411 .get("s1")
3412 .and_then(plugin::OperationInfo::parent_id),
3413 Some("exec-1")
3414 );
3415 assert_eq!(
3416 operations.get("c1").map(plugin::OperationInfo::parent_id),
3417 Some(None)
3418 );
3419 assert_eq!(
3420 infos.first().unwrap().execution_start_timestamp(),
3421 Some(std::time::UNIX_EPOCH + Duration::from_secs(7))
3422 );
3423 assert!(!infos.first().unwrap().is_first_invocation());
3424 }
3425
3426 #[tokio::test]
3427 async fn invocation_info_reads_lambda_context() {
3428 let payload = || {
3429 envelope_with(&serde_json::json!([{
3430 "Id": "exec-1",
3431 "Type": "EXECUTION",
3432 "Status": "STARTED",
3433 "ExecutionDetails": { "InputPayload": "{\"k\":1}" }
3434 }]))
3435 };
3436 let mut lambda_ctx = lambda_runtime::Context::default();
3437 lambda_ctx.request_id = "r-1".to_owned();
3438 lambda_ctx.xray_trace_id = Some("Root=1-abc".to_owned());
3439 let infos = infos_for(payload(), lambda_ctx).await;
3440 let info = infos.first().unwrap();
3441 assert_eq!(info.request_id(), "r-1");
3442 assert_eq!(info.x_ray_trace_id(), Some("Root=1-abc"));
3443 assert_eq!(info.execution_arn(), "arn:test");
3444 assert_eq!(info.execution_input(), "{\"k\":1}");
3445 assert!(info.is_first_invocation());
3446
3447 let mut lambda_ctx = lambda_runtime::Context::default();
3448 lambda_ctx.xray_trace_id = None;
3449 let infos = infos_for(payload(), lambda_ctx).await;
3450 assert_eq!(infos.first().unwrap().x_ray_trace_id(), None);
3451 }
3452
3453 #[tokio::test]
3454 async fn invocation_info_defaults_input() {
3455 let payload = envelope_with(&serde_json::json!([{
3456 "Id": "exec-1",
3457 "Type": "EXECUTION",
3458 "Status": "STARTED"
3459 }]));
3460 let infos = infos_for(payload, lambda_runtime::Context::default()).await;
3461 assert_eq!(infos.first().unwrap().execution_input(), "{}");
3462 }
3463
3464 struct PanickingDropPlugin;
3466
3467 impl plugin::Plugin for PanickingDropPlugin {}
3468
3469 impl Drop for PanickingDropPlugin {
3470 #[expect(clippy::panic)] fn drop(&mut self) {
3472 panic!("plugin drop");
3473 }
3474 }
3475
3476 #[derive(Debug)]
3478 struct PanickingDropError;
3479
3480 impl std::fmt::Display for PanickingDropError {
3481 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3482 f.write_str("drop err")
3483 }
3484 }
3485
3486 impl std::error::Error for PanickingDropError {}
3487
3488 impl Drop for PanickingDropError {
3489 #[expect(clippy::panic)] fn drop(&mut self) {
3491 panic!("error drop");
3492 }
3493 }
3494
3495 struct PanickingDropFactory {
3498 failing: bool,
3499 }
3500
3501 impl plugin::PluginFactory for PanickingDropFactory {
3502 fn create_plugin(
3503 &self,
3504 _info: &plugin::InvocationInfo,
3505 ) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
3506 if self.failing {
3507 Err(Box::new(PanickingDropError))
3508 } else {
3509 Ok(Some(Box::new(PanickingDropPlugin)))
3510 }
3511 }
3512 }
3513
3514 fn service_with_factory<F, Fut>(
3517 factory: PanickingDropFactory,
3518 handler: F,
3519 ) -> (
3520 impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> BoxedInvocationFuture,
3521 lambda_runtime::LambdaEvent<InvocationPayload>,
3522 )
3523 where
3524 F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
3525 Fut: Future<Output = Result<serde_json::Value, BoxError>> + Send,
3526 {
3527 let options = Options::builder()
3528 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
3529 Vec::new(),
3530 )))
3531 .plugin(factory)
3532 .build()
3533 .unwrap();
3534 let payload = envelope_with(&serde_json::json!([{
3535 "Id": "exec-1",
3536 "Type": "EXECUTION",
3537 "Status": "STARTED",
3538 "ExecutionDetails": { "InputPayload": "{}" }
3539 }]));
3540 let event = lambda_runtime::LambdaEvent::new(
3541 InvocationPayload(payload),
3542 lambda_runtime::Context::default(),
3543 );
3544 (wrap(handler, options), event)
3545 }
3546
3547 #[tokio::test]
3550 async fn plugin_drop_panic_keeps_successful_outcome() {
3551 let (service, event) = service_with_factory(
3552 PanickingDropFactory { failing: false },
3553 |_event, _ctx| async move { Ok(serde_json::json!("done")) },
3554 );
3555 let json = service(event).await.unwrap().0;
3556 assert_eq!(
3557 json.get("Status"),
3558 Some(&serde_json::json!("SUCCEEDED")),
3559 "{json}"
3560 );
3561 assert_eq!(
3562 json.get("Result"),
3563 Some(&serde_json::json!("\"done\"")),
3564 "{json}"
3565 );
3566 }
3567
3568 #[tokio::test]
3572 #[expect(clippy::disallowed_methods)]
3575 async fn plugin_drop_panic_while_handler_panics_resumes_handler_panic() {
3576 #[expect(clippy::panic)] fn boom() -> Result<serde_json::Value, BoxError> {
3578 panic!("root boom")
3579 }
3580 let (service, event) = service_with_factory(
3581 PanickingDropFactory { failing: false },
3582 |_event, _ctx| async move { boom() },
3583 );
3584 let join_err = tokio::spawn(service(event))
3585 .await
3586 .expect_err("the handler panic leaves the service");
3587 assert!(join_err.is_panic(), "a panic, not a cancellation");
3588 let payload = join_err.into_panic();
3589 assert_eq!(payload.downcast_ref::<&str>(), Some(&"root boom"));
3590 }
3591
3592 #[tokio::test]
3595 #[expect(clippy::disallowed_methods)]
3598 async fn factory_error_destructor_panic_fails_the_invocation() {
3599 let (service, event) = service_with_factory(
3600 PanickingDropFactory { failing: true },
3601 |_event, _ctx| async move { Ok(serde_json::Value::Null) },
3602 );
3603 let join_err = tokio::spawn(service(event))
3604 .await
3605 .expect_err("the destructor panic leaves the service");
3606 assert!(join_err.is_panic(), "a panic, not a cancellation");
3607 let payload = join_err.into_panic();
3608 assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
3609 }
3610
3611 use plugin::test_support::{self as hooks, Hooks};
3614
3615 #[tokio::test(start_paused = true)]
3618 #[expect(clippy::panic)] async fn end_hook_cut_off_before_deadline() {
3620 for _attempt in 0..5 {
3621 let (buffer, _guard) = hooks::capture();
3622 let plugin = Hooks::default().on_end(|_| Box::pin(std::future::pending()));
3623 let (service, event) = hooks::service(
3624 vec![plugin],
3625 hooks::deadline_in(1_000),
3626 |_e, _ctx| async move { Ok(serde_json::json!("ok")) },
3627 );
3628 let json = service(event).await.unwrap().0;
3629 assert_eq!(json.get("Status"), Some(&serde_json::json!("SUCCEEDED")));
3630 let warnings = hooks::warnings(&buffer);
3631 let [_] = hooks::with_message(
3632 &warnings,
3633 "plugin on_invocation_end hooks did not finish before the invocation deadline; \
3634 the response returns without them",
3635 )[..] else {
3636 continue;
3637 };
3638 return;
3639 }
3640 panic!("no attempt captured the warning");
3641 }
3642
3643 #[tokio::test(start_paused = true)]
3644 async fn end_hook_unbounded_without_deadline() {
3645 let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
3646 let plugin = {
3647 let flag = std::sync::Arc::clone(&flag);
3648 Hooks::default().on_end(move |_| {
3649 let flag = std::sync::Arc::clone(&flag);
3650 Box::pin(async move {
3651 tokio::time::sleep(Duration::from_millis(100)).await;
3652 flag.store(true, std::sync::atomic::Ordering::SeqCst);
3653 Ok(())
3654 })
3655 })
3656 };
3657 let (service, event) = hooks::service(vec![plugin], 0, |_e, _ctx| async move {
3658 Ok(serde_json::Value::Null)
3659 });
3660 service(event).await.unwrap();
3661 assert!(flag.load(std::sync::atomic::Ordering::SeqCst));
3662 }
3663
3664 #[tokio::test]
3668 async fn end_info_projects_operations_once() {
3669 async fn thousand_steps(
3670 _event: serde_json::Value,
3671 ctx: DurableContext,
3672 ) -> Result<serde_json::Value, BoxError> {
3673 for i in 1..=1_000_u32 {
3674 ctx.step(move |_| async move { Ok(i) }).await?;
3675 }
3676 Ok(serde_json::Value::Null)
3677 }
3678 let recorded = || {
3681 (1..=1_000_u32)
3682 .map(|i| {
3683 serde_json::json!({
3684 "Id": engine::compute_wire_id_public(&i.to_string()),
3685 "Type": "STEP",
3686 "SubType": "Step",
3687 "Status": "SUCCEEDED",
3688 "StepDetails": { "Result": i.to_string() }
3689 })
3690 })
3691 .collect::<Vec<_>>()
3692 };
3693 let mut growths = Vec::new();
3694 for count in [1, 3] {
3695 let plugins: Vec<Hooks> = (0..count).map(|_| Hooks::default()).collect();
3696 let (service, event) =
3697 hooks::service_with(plugins.clone(), 0, recorded(), thousand_steps);
3698 let before = engine::projections();
3699 service(event).await.unwrap();
3700 growths.push(engine::projections() - before);
3701 for plugin in &plugins {
3702 let ends = plugin.ends();
3703 let [end] = ends.as_slice() else {
3704 unreachable!("one end info per plugin, got {}", ends.len());
3705 };
3706 assert_eq!(end.operations().len(), 1_001);
3707 }
3708 }
3709 let [one, three] = growths.as_slice() else {
3710 unreachable!("two runs");
3711 };
3712 assert_eq!(one, three);
3713 }
3714
3715 fn json_str<'a>(json: &'a serde_json::Value, pointer: &str) -> Option<&'a str> {
3717 json.pointer(pointer).and_then(serde_json::Value::as_str)
3718 }
3719
3720 async fn finish_with_failing_flush<R>(
3725 root: impl FnOnce(DurableContext) -> R,
3726 ) -> (
3727 serde_json::Value,
3728 plugin::InvocationEndInfo,
3729 Vec<aws_sdk_lambda::types::OperationUpdate>,
3730 )
3731 where
3732 R: Future<Output = Result<String, WireError>> + Send + 'static,
3733 {
3734 let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
3735 client.enqueue_checkpoint_response(client::TestResponse::NonRetryableError(
3736 "flush down".to_owned(),
3737 ));
3738 let hook = Hooks::default();
3739 let plugins = hooks::plugins_of(hook.clone());
3740 let ctx = DurableContext::new_root_with_client_and_defaults(
3741 "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
3742 lambda_runtime::Context::default(),
3743 std::sync::Arc::new(engine::CheckpointLog::empty()),
3744 std::sync::Arc::clone(&client) as std::sync::Arc<dyn ExecutionClient>,
3745 "t0".to_owned(),
3746 Duration::from_secs(60),
3747 0,
3748 None,
3749 );
3750 let update = aws_sdk_lambda::types::OperationUpdate::builder()
3751 .id("s")
3752 .name("s")
3753 .r#type(aws_sdk_lambda::types::OperationType::Step)
3754 .action(aws_sdk_lambda::types::OperationAction::Succeed)
3755 .payload("1")
3756 .build()
3757 .expect("all required fields set");
3758 ctx.buffer_unflushed(vec![update]);
3759 let end_log = engine::CheckpointLog::empty();
3760 let finished = finish_invocation(
3761 root(ctx.clone()),
3762 task::TaskContext::current(),
3763 std::sync::Arc::clone(ctx.task_ownership()),
3764 std::sync::Arc::clone(ctx.coordinator()),
3765 &ctx,
3766 None,
3767 Some(&plugins),
3768 &end_log,
3769 )
3770 .await;
3771 let Finished::Respond(response, _late) = finished else {
3772 unreachable!("the invocation answers");
3773 };
3774 let ends = hook.ends();
3775 let [end] = ends.as_slice() else {
3776 unreachable!("one end hook, got {}", ends.len());
3777 };
3778 (
3779 response.expect("an envelope").0,
3780 end.clone(),
3781 client.recorded_updates(),
3782 )
3783 }
3784
3785 #[tokio::test]
3791 async fn flush_failure_decides_over_handler_error() {
3792 let (json, end, updates) = finish_with_failing_flush(|_ctx| async {
3793 Err(WireError::new(Some("Error"), Some("boom")))
3794 })
3795 .await;
3796 assert_eq!(json_str(&json, "/Status"), Some("FAILED"), "{json}");
3797 assert_eq!(
3798 json_str(&json, "/Error/ErrorType"),
3799 Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
3800 );
3801 assert_eq!(end.status(), InvocationStatus::Failed);
3802 assert_eq!(
3803 end.execution_error().and_then(WireError::error_type),
3804 Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
3805 );
3806 assert_eq!(updates.len(), 1, "only the flushed SUCCEED: {updates:?}");
3807 assert!(
3808 updates
3809 .iter()
3810 .all(|u| u.action() != &aws_sdk_lambda::types::OperationAction::Fail),
3811 "{updates:?}"
3812 );
3813 }
3814
3815 #[tokio::test]
3818 async fn flush_failure_after_a_winning_fatal_keeps_the_fatal() {
3819 let (json, end, _updates) = finish_with_failing_flush(|ctx| async move {
3820 ctx.coordinator()
3821 .record_fatal("UsageError".to_owned(), "misuse".to_owned());
3822 std::future::pending().await
3823 })
3824 .await;
3825 assert_eq!(json_str(&json, "/Status"), Some("FAILED"), "{json}");
3826 assert_eq!(json_str(&json, "/Error/ErrorType"), Some("UsageError"));
3827 assert_eq!(
3828 end.execution_error().and_then(WireError::error_type),
3829 Some("UsageError")
3830 );
3831 }
3832
3833 #[test]
3837 fn late_panics_fail_only_a_response_without_a_fatal_error() {
3838 use crate::plugin::LatePanics;
3839 let ok = Ok(InvocationResponse(serde_json::Value::Null));
3840 let err = Err(lambda_runtime::Error::from("fault"));
3841 assert_eq!(late_panics(Some(&ok), false), LatePanics::Fail);
3842 assert_eq!(late_panics(Some(&err), false), LatePanics::Discard);
3843 assert_eq!(late_panics(None, false), LatePanics::Discard);
3844 assert_eq!(late_panics(Some(&ok), true), LatePanics::Discard);
3845 }
3846
3847 #[expect(clippy::disallowed_methods)]
3849 async fn panic_payload_of<Fut>(future: Fut) -> Box<dyn std::any::Any + Send>
3852 where
3853 Fut: Future + Send + 'static,
3854 Fut::Output: Send,
3855 {
3856 match tokio::spawn(future).await {
3857 Ok(_) => unreachable!("the service must panic"),
3858 Err(join_err) => join_err.into_panic(),
3859 }
3860 }
3861
3862 #[tokio::test]
3863 #[expect(clippy::panic)] async fn end_hook_retrying_after_panic() {
3865 let messages = [
3866 "boom",
3867 "boom",
3868 "the invocation panicked with a payload that is not a string",
3869 ];
3870 for (case, message) in messages.into_iter().enumerate() {
3871 let plugin = Hooks::default();
3872 let (service, event) =
3873 hooks::service(vec![plugin.clone()], 0, move |_e, _ctx| async move {
3874 match case {
3875 0 => panic!("boom"),
3876 1 => panic!("{}", String::from("boom")),
3877 _ => std::panic::panic_any(42_u32),
3878 }
3879 });
3880 let resumed = panic_payload_of(service(event)).await;
3881 let original = match case {
3882 0 => resumed.downcast_ref::<&str>() == Some(&"boom"),
3883 1 => resumed.downcast_ref::<String>().map(String::as_str) == Some("boom"),
3884 _ => resumed.downcast_ref::<u32>() == Some(&42),
3885 };
3886 assert!(original, "case {case}: the original payload leaves the SDK");
3887 let ends = plugin.ends();
3888 let [end] = ends.as_slice() else {
3889 unreachable!("one end info, got {}", ends.len());
3890 };
3891 assert_eq!(end.status(), InvocationStatus::Retrying);
3892 assert_eq!(end.execution_result(), None);
3893 let error = end.execution_error().expect("a panic end has an error");
3894 assert_eq!(error.error_type(), None);
3895 assert_eq!(error.error_message(), Some(message));
3896 }
3897 }
3898
3899 #[tokio::test]
3900 #[expect(clippy::panic)] async fn end_hook_after_panic_runs_after_cleanup() {
3902 struct Guard(std::sync::Arc<std::sync::atomic::AtomicBool>);
3903 impl Drop for Guard {
3904 fn drop(&mut self) {
3905 self.0.store(true, std::sync::atomic::Ordering::SeqCst);
3906 }
3907 }
3908 let dropped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
3909 let seen = std::sync::Arc::new(std::sync::Mutex::new(None));
3910 let plugin = {
3911 let dropped = std::sync::Arc::clone(&dropped);
3912 let seen = std::sync::Arc::clone(&seen);
3913 Hooks::default().on_end(move |_| {
3914 *seen.lock().unwrap() = Some(dropped.load(std::sync::atomic::Ordering::SeqCst));
3915 Box::pin(std::future::ready(Ok(())))
3916 })
3917 };
3918 let (service, event) = hooks::service(vec![plugin], 0, move |_e, ctx| {
3919 let dropped = std::sync::Arc::clone(&dropped);
3920 async move {
3921 let _held = ctx
3922 .step(move |_| {
3923 let guard = Guard(std::sync::Arc::clone(&dropped));
3924 async move {
3925 let _guard = guard;
3926 std::future::pending::<()>().await;
3927 Ok(0_u32)
3928 }
3929 })
3930 .spawn();
3931 tokio::time::sleep(Duration::from_millis(10)).await;
3932 panic!("boom")
3933 }
3934 });
3935 let payload = panic_payload_of(service(event)).await;
3936 assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
3937 assert_eq!(*seen.lock().unwrap(), Some(true));
3938 }
3939
3940 #[tokio::test]
3944 #[expect(clippy::panic)] async fn plugins_drop_before_the_panic_resumes() {
3946 struct Probe(std::sync::Arc<std::sync::Mutex<Option<bool>>>);
3947 impl plugin::Plugin for Probe {}
3948 impl Drop for Probe {
3949 fn drop(&mut self) {
3950 *self.0.lock().unwrap() = Some(std::thread::panicking());
3951 }
3952 }
3953 struct ProbeFactory(std::sync::Arc<std::sync::Mutex<Option<bool>>>);
3954 impl plugin::PluginFactory for ProbeFactory {
3955 fn create_plugin(
3956 &self,
3957 _info: &plugin::InvocationInfo,
3958 ) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
3959 Ok(Some(Box::new(Probe(std::sync::Arc::clone(&self.0)))))
3960 }
3961 }
3962 let unwinding = std::sync::Arc::new(std::sync::Mutex::new(None));
3963 let options = Options::builder()
3964 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
3965 Vec::new(),
3966 )))
3967 .plugin(ProbeFactory(std::sync::Arc::clone(&unwinding)))
3968 .build()
3969 .unwrap();
3970 let service = wrap(
3971 |_e: serde_json::Value, _ctx: DurableContext| async move {
3972 panic!("boom");
3973 #[expect(unreachable_code)] Ok::<serde_json::Value, BoxError>(serde_json::Value::Null)
3975 },
3976 options,
3977 );
3978 let event = lambda_runtime::LambdaEvent::new(
3979 InvocationPayload(envelope_with(&serde_json::json!([{
3980 "Id": "exec-1",
3981 "Type": "EXECUTION",
3982 "Status": "STARTED",
3983 "ExecutionDetails": { "InputPayload": "{}" }
3984 }]))),
3985 lambda_runtime::Context::default(),
3986 );
3987 let payload = panic_payload_of(service(event)).await;
3988 assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
3989 assert_eq!(*unwinding.lock().unwrap(), Some(false));
3990 }
3991
3992 #[tokio::test]
3993 #[expect(clippy::panic)] async fn panicking_end_hook_keeps_the_original_panic() {
3995 for _attempt in 0..5 {
3996 let (buffer, _guard) = hooks::capture();
3997 let faulty = Hooks::default().named("Faulty").on_end(|_| panic!("hook"));
3998 let healthy = Hooks::default();
3999 let (service, event) =
4000 hooks::service(vec![faulty, healthy.clone()], 0, |_e, _ctx| async move {
4001 panic!("boom")
4002 });
4003 let payload = panic_payload_of(service(event)).await;
4004 assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
4005 let ends = healthy.ends();
4006 let [end] = ends.as_slice() else {
4007 unreachable!("one end info, got {}", ends.len());
4008 };
4009 assert_eq!(end.status(), InvocationStatus::Retrying);
4010 let warnings = hooks::warnings(&buffer);
4011 let [warning] =
4012 hooks::with_message(&warnings, "plugin hook panicked; the panic is ignored")[..]
4013 else {
4014 continue;
4015 };
4016 assert!(
4017 hooks::field(warning, "plugin").ends_with("Faulty (plugins[0])"),
4018 "{warning}"
4019 );
4020 assert_eq!(hooks::field(warning, "hook"), "on_invocation_end");
4021 return;
4022 }
4023 panic!("no attempt captured the warning");
4024 }
4025
4026 #[derive(Clone, Copy, PartialEq)]
4032 enum ClaimAt {
4033 AfterNext,
4034 OperationStart,
4035 }
4036
4037 type ContextSlot = std::sync::Arc<std::sync::Mutex<Option<DurableContext>>>;
4038
4039 #[derive(Clone)]
4040 struct ClaimPlugin {
4041 at: ClaimAt,
4042 slot: ContextSlot,
4043 claims: std::sync::Arc<std::sync::atomic::AtomicUsize>,
4044 armed: std::sync::Arc<std::sync::atomic::AtomicBool>,
4047 }
4048
4049 impl ClaimPlugin {
4050 fn new(at: ClaimAt) -> Self {
4051 Self {
4052 at,
4053 slot: std::sync::Arc::default(),
4054 claims: std::sync::Arc::default(),
4055 armed: std::sync::Arc::default(),
4056 }
4057 }
4058
4059 fn claims(&self) -> usize {
4060 self.claims.load(std::sync::atomic::Ordering::SeqCst)
4061 }
4062
4063 fn claim_once(&self, ctx: &DurableContext) {
4064 if self
4065 .claims
4066 .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
4067 == 0
4068 {
4069 drop(ctx.wait(Duration::from_secs(1)));
4070 }
4071 }
4072 }
4073
4074 impl plugin::Plugin for ClaimPlugin {
4075 fn on_operation_start(&self, _info: &plugin::OperationInfo) -> plugin::HookWork {
4076 if self.at == ClaimAt::OperationStart
4077 && let Some(ctx) = self.slot.lock().unwrap().clone()
4078 {
4079 self.claim_once(&ctx);
4080 }
4081 Ok(None)
4082 }
4083
4084 fn wrap_invocation(&self, _info: &plugin::InvocationInfo, next: &mut dyn FnMut()) {
4085 next();
4086 let ctx = self.slot.lock().unwrap().clone();
4087 if self.at == ClaimAt::AfterNext
4088 && let Some(ctx) = ctx
4089 && ctx.coordinator().phase() != coordinator::Phase::Running
4090 && self.armed.load(std::sync::atomic::Ordering::SeqCst)
4091 {
4092 self.claim_once(&ctx);
4093 }
4094 }
4095 }
4096
4097 impl plugin::PluginFactory for ClaimPlugin {
4098 fn create_plugin(
4099 &self,
4100 _info: &plugin::InvocationInfo,
4101 ) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
4102 Ok(Some(Box::new(self.clone())))
4103 }
4104 }
4105
4106 #[derive(Debug)]
4108 struct LateDropError;
4109
4110 impl std::fmt::Display for LateDropError {
4111 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4112 f.write_str("late drop error")
4113 }
4114 }
4115
4116 impl std::error::Error for LateDropError {}
4117
4118 impl Drop for LateDropError {
4119 #[expect(clippy::panic)] fn drop(&mut self) {
4121 panic!("late drop");
4122 }
4123 }
4124
4125 fn first_invocation_event() -> lambda_runtime::LambdaEvent<InvocationPayload> {
4127 lambda_runtime::LambdaEvent::new(
4128 InvocationPayload(envelope_with(&serde_json::json!([{
4129 "Id": "exec-1",
4130 "Type": "EXECUTION",
4131 "Status": "STARTED",
4132 "ExecutionDetails": { "InputPayload": "{}" }
4133 }]))),
4134 lambda_runtime::Context::default(),
4135 )
4136 }
4137
4138 #[tokio::test]
4147 #[expect(clippy::disallowed_methods)]
4150 async fn late_panic_after_post_drive_claim_fails_the_invocation() {
4151 let (buffer, _guard) = hooks::capture();
4152 let claimer = ClaimPlugin::new(ClaimAt::AfterNext);
4153 let claims_at_end = std::sync::Arc::new(std::sync::Mutex::new(None));
4154 let failing = {
4155 let claimer = claimer.clone();
4156 let claims_at_end = std::sync::Arc::clone(&claims_at_end);
4157 Hooks::default().on_end(move |_| {
4158 *claims_at_end.lock().unwrap() = Some(claimer.claims());
4159 Box::pin(async { Err(Box::new(LateDropError) as BoxError) })
4160 })
4161 };
4162 let options = Options::builder()
4163 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
4164 Vec::new(),
4165 )))
4166 .plugin(claimer.clone())
4167 .plugin(failing)
4168 .build()
4169 .unwrap();
4170 let slot = std::sync::Arc::clone(&claimer.slot);
4171 let service = wrap(
4172 move |_e: serde_json::Value, ctx: DurableContext| {
4173 *slot.lock().unwrap() = Some(ctx);
4174 async { Ok::<_, BoxError>(serde_json::json!("done")) }
4175 },
4176 options,
4177 );
4178 let joined = tokio::spawn(test_hooks::PENDING_AFTER_DRIVE.scope(
4181 std::sync::Arc::clone(&claimer.armed),
4182 service(first_invocation_event()),
4183 ))
4184 .await;
4185 let payload = match joined {
4186 Ok(response) => unreachable!(
4187 "the late panic must fail the invocation: {:?}",
4188 response.map(|r| r.0)
4189 ),
4190 Err(join_err) => join_err.into_panic(),
4191 };
4192 assert_eq!(payload.downcast_ref::<&str>(), Some(&"late drop"));
4193 assert_eq!(
4194 *claims_at_end.lock().unwrap(),
4195 Some(1),
4196 "the claim ran after next() in the post-drive pending poll, before the end hooks"
4197 );
4198 let ctx = claimer
4199 .slot
4200 .lock()
4201 .unwrap()
4202 .take()
4203 .expect("the handler ran");
4204 let fatal = ctx
4205 .coordinator()
4206 .fatal_error()
4207 .map(|fatal| fatal.error_message);
4208 assert!(
4209 fatal
4210 .as_deref()
4211 .is_some_and(|m| m.contains("inside plugin code")),
4212 "the claim records the context usage error: {fatal:?}"
4213 );
4214 let warnings = hooks::warnings(&buffer);
4217 assert_eq!(
4218 hooks::with_message(&warnings, "plugin hook failed; the error is ignored").len(),
4219 1,
4220 "{warnings:?}"
4221 );
4222 assert_eq!(warnings.len(), 1, "no claim WARN: {warnings:?}");
4223 }
4224
4225 #[tokio::test]
4232 #[expect(clippy::disallowed_methods)]
4235 async fn fatal_after_driver_arbitration_keeps_the_late_panic() {
4236 let failing = Hooks::default()
4237 .on_end(|_| Box::pin(async { Err(Box::new(LateDropError) as BoxError) }));
4238 let options = Options::builder()
4239 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
4240 Vec::new(),
4241 )))
4242 .plugin(failing)
4243 .build()
4244 .unwrap();
4245 let slot: ContextSlot = std::sync::Arc::default();
4246 let service = {
4247 let slot = std::sync::Arc::clone(&slot);
4248 wrap(
4249 move |_e: serde_json::Value, ctx: DurableContext| {
4250 *slot.lock().unwrap() = Some(ctx);
4251 async { Ok::<_, BoxError>(serde_json::json!("done")) }
4252 },
4253 options,
4254 )
4255 };
4256 let hook: std::sync::Arc<dyn Fn(&coordinator::InvocationCoordinator) + Send + Sync> = {
4257 let slot = std::sync::Arc::clone(&slot);
4258 std::sync::Arc::new(move |_coordinator| {
4259 let ctx = slot.lock().unwrap().clone().expect("the handler ran");
4260 plugin::call_plugin(|| drop(ctx.wait(Duration::from_secs(1))))
4262 .expect("the claim does not panic");
4263 })
4264 };
4265 let joined = tokio::spawn(
4266 driver::test_hooks::AFTER_ARBITRATION.scope(hook, service(first_invocation_event())),
4267 )
4268 .await;
4269 let payload = match joined {
4270 Ok(response) => unreachable!(
4271 "the late panic must fail the invocation: {:?}",
4272 response.map(|r| r.0)
4273 ),
4274 Err(join_err) => join_err.into_panic(),
4275 };
4276 assert_eq!(payload.downcast_ref::<&str>(), Some(&"late drop"));
4277 let ctx = slot.lock().unwrap().take().expect("the handler ran");
4278 let fatal = ctx
4279 .coordinator()
4280 .fatal_error()
4281 .map(|fatal| fatal.error_message);
4282 assert!(
4283 fatal
4284 .as_deref()
4285 .is_some_and(|m| m.contains("inside plugin code")),
4286 "the claim after the arbitration records the context usage error: {fatal:?}"
4287 );
4288 }
4289
4290 #[tokio::test]
4295 async fn claim_in_end_hook_leaves_response_unchanged() {
4296 let slot: ContextSlot = std::sync::Arc::default();
4297 let kept: ContextSlot = std::sync::Arc::default();
4298 let plugin = {
4299 let slot = std::sync::Arc::clone(&slot);
4300 Hooks::default().on_end(move |_| {
4301 if let Some(ctx) = slot.lock().unwrap().as_ref() {
4302 drop(ctx.wait(Duration::from_secs(1)));
4303 }
4304 Box::pin(std::future::ready(Ok(())))
4305 })
4306 };
4307 let handler_slot = std::sync::Arc::clone(&slot);
4308 let handler_kept = std::sync::Arc::clone(&kept);
4309 let (service, event) = hooks::service(vec![plugin], 0, move |_e, ctx| {
4310 *handler_slot.lock().unwrap() = Some(ctx.clone());
4311 *handler_kept.lock().unwrap() = Some(ctx);
4312 async move { Ok(serde_json::json!("done")) }
4313 });
4314 let json = service(event).await.unwrap().0;
4315 let ctx = kept.lock().unwrap().take().expect("the handler ran");
4316 drop(slot.lock().unwrap().take());
4317 let fatal = ctx
4318 .coordinator()
4319 .fatal_error()
4320 .map(|fatal| fatal.error_message);
4321 assert!(
4322 fatal
4323 .as_deref()
4324 .is_some_and(|m| m.contains("inside plugin code")),
4325 "the end-hook claim records the context usage error: {fatal:?}"
4326 );
4327 assert_eq!(
4328 json.get("Status"),
4329 Some(&serde_json::json!("SUCCEEDED")),
4330 "{json}"
4331 );
4332 assert_eq!(
4333 json.get("Result"),
4334 Some(&serde_json::json!("\"done\"")),
4335 "{json}"
4336 );
4337 }
4338
4339 #[tokio::test]
4343 async fn a_claim_while_the_handler_runs_fails_the_execution() {
4344 let claimer = ClaimPlugin::new(ClaimAt::OperationStart);
4345 let options = Options::builder()
4346 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
4347 Vec::new(),
4348 )))
4349 .plugin(claimer.clone())
4350 .build()
4351 .unwrap();
4352 let slot = std::sync::Arc::clone(&claimer.slot);
4353 let service = wrap(
4354 move |_e: serde_json::Value, ctx: DurableContext| {
4355 *slot.lock().unwrap() = Some(ctx.clone());
4356 async move {
4357 let value = ctx.step(|_| async { Ok(1_u32) }).name("s").await?;
4358 Ok::<_, BoxError>(serde_json::json!(value))
4359 }
4360 },
4361 options,
4362 );
4363 let json = service(first_invocation_event()).await.unwrap().0;
4364 drop(claimer.slot.lock().unwrap().take());
4365 assert_eq!(claimer.claims(), 1, "on_operation_start claimed once");
4366 assert_eq!(
4367 json.get("Status"),
4368 Some(&serde_json::json!("FAILED")),
4369 "{json}"
4370 );
4371 assert!(json.to_string().contains("inside plugin code"), "{json}");
4372 }
4373
4374 type Log = std::sync::Arc<std::sync::Mutex<Vec<String>>>;
4375
4376 struct InvocationWrapLog(Log);
4378
4379 impl plugin::Plugin for InvocationWrapLog {
4380 fn wrap_invocation(&self, _info: &plugin::InvocationInfo, next: &mut dyn FnMut()) {
4381 self.0.lock().unwrap().push("inv enter".to_owned());
4382 next();
4383 self.0.lock().unwrap().push("inv exit".to_owned());
4384 }
4385 }
4386
4387 impl plugin::PluginFactory for InvocationWrapLog {
4388 fn create_plugin(
4389 &self,
4390 _info: &plugin::InvocationInfo,
4391 ) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
4392 Ok(Some(Box::new(Self(std::sync::Arc::clone(&self.0)))))
4393 }
4394 }
4395
4396 struct PanicsOnDrop(Log);
4398
4399 impl Drop for PanicsOnDrop {
4400 #[expect(clippy::panic)] fn drop(&mut self) {
4402 self.0.lock().unwrap().push("guard drop".to_owned());
4403 panic!("guard drop");
4404 }
4405 }
4406
4407 #[tokio::test]
4413 async fn pending_invocation_future_destructor_panic() {
4414 let log: Log = std::sync::Arc::default();
4415 let reached = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
4416 let held = std::sync::Arc::new(test_hooks::HeldGuard {
4417 reached: std::sync::Arc::clone(&reached),
4418 make: Box::new({
4419 let log = std::sync::Arc::clone(&log);
4420 move || Box::new(PanicsOnDrop(std::sync::Arc::clone(&log))) as Box<dyn Send>
4421 }),
4422 });
4423 let options = Options::builder()
4424 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
4425 Vec::new(),
4426 )))
4427 .plugin(InvocationWrapLog(std::sync::Arc::clone(&log)))
4428 .build()
4429 .unwrap();
4430 let service = wrap(
4431 |_e: serde_json::Value, _ctx: DurableContext| async {
4432 Ok::<_, BoxError>(serde_json::json!("done"))
4433 },
4434 options,
4435 );
4436 let mut invocation = Box::pin(
4437 test_hooks::PENDING_HOLDING_GUARD.scope(held, service(first_invocation_event())),
4438 );
4439 tokio::time::timeout(
4442 Duration::from_secs(5),
4443 std::future::poll_fn(|cx| {
4444 assert!(
4445 invocation.as_mut().poll(cx).is_pending(),
4446 "the invocation stays pending at the pending point"
4447 );
4448 if reached.load(std::sync::atomic::Ordering::SeqCst) {
4449 std::task::Poll::Ready(())
4450 } else {
4451 std::task::Poll::Pending
4452 }
4453 }),
4454 )
4455 .await
4456 .expect("the post-start future reaches the pending point within 5 s");
4457 log.lock().unwrap().push("dropping".to_owned());
4458 let dropped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(invocation)));
4459 let payload = dropped.expect_err("the destructor panic propagates");
4460 assert_eq!(payload.downcast_ref::<&str>(), Some(&"guard drop"));
4461 let log = log.lock().unwrap().clone();
4462 let at = log.iter().position(|l| l == "dropping").expect("logged");
4463 assert_eq!(
4464 log.get(at..).expect("in range"),
4465 ["dropping", "inv enter", "guard drop", "inv exit"],
4466 "{log:?}"
4467 );
4468 }
4469
4470 #[tokio::test]
4473 async fn failed_input_calls_no_factory() {
4474 let factory = RecordingFactory::default();
4475 let options = Options::builder()
4476 .execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
4477 Vec::new(),
4478 )))
4479 .plugin(factory.clone())
4480 .build()
4481 .unwrap();
4482 let service = wrap(
4483 |_event: u32, _ctx: DurableContext| async move { Ok::<u32, BoxError>(0) },
4484 options,
4485 );
4486 let payload = envelope_with(&serde_json::json!([{
4487 "Id": "exec-1",
4488 "Type": "EXECUTION",
4489 "ExecutionDetails": { "InputPayload": "\"not a number\"" }
4490 }]));
4491 let result = service(lambda_runtime::LambdaEvent::new(
4492 InvocationPayload(payload),
4493 lambda_runtime::Context::default(),
4494 ))
4495 .await;
4496 assert!(result.is_err());
4497 assert!(factory.0.lock().unwrap().is_empty());
4498 }
4499
4500 #[test]
4501 fn parse_envelope_no_envelope_returns_none() {
4502 let payload = serde_json::json!({ "order_id": "abc-123" });
4504 let result = parse_envelope(&payload).unwrap();
4505 assert!(result.is_none(), "non-envelope payload should return None");
4506 }
4507
4508 #[test]
4509 fn extract_customer_input_from_valid_envelope() {
4510 let payload = serde_json::json!({
4511 "DurableExecutionArn": "arn:test",
4512 "CheckpointToken": "tok",
4513 "InitialExecutionState": {
4514 "Operations": [{
4515 "Id": "execution",
4516 "Type": "EXECUTION",
4517 "ExecutionDetails": {
4518 "InputPayload": "\"hello\""
4519 }
4520 }]
4521 }
4522 });
4523 let input = inline_input_payload(&payload).unwrap().into_payload();
4524 let result: Result<String, _> = extract_customer_input(input.as_deref());
4525 assert_eq!(result.unwrap(), "hello");
4526 }
4527
4528 #[test]
4529 fn extract_customer_input_defaults_to_empty_object() {
4530 let payload = serde_json::json!({
4533 "DurableExecutionArn": "arn:test",
4534 "CheckpointToken": "tok",
4535 "InitialExecutionState": {
4536 "Operations": [{ "Id": "execution", "Type": "EXECUTION" }]
4537 }
4538 });
4539 let input = inline_input_payload(&payload).unwrap();
4540 assert_eq!(input, client::PageInput::Execution(None));
4541 let result: serde_json::Value =
4542 extract_customer_input(input.into_payload().as_deref()).unwrap();
4543 assert_eq!(result, serde_json::json!({}));
4544 }
4545
4546 #[test]
4549 fn inline_input_payload_cases() {
4550 let envelope = |ops: serde_json::Value| {
4551 serde_json::json!({
4552 "DurableExecutionArn": "arn:test",
4553 "CheckpointToken": "tok",
4554 "InitialExecutionState": { "Operations": ops }
4555 })
4556 };
4557 let step_only = envelope(serde_json::json!([{ "Id": "s1", "Type": "STEP" }]));
4558 assert_eq!(
4559 inline_input_payload(&step_only).unwrap(),
4560 client::PageInput::NoExecutionOperation
4561 );
4562 let no_operations = serde_json::json!({
4563 "DurableExecutionArn": "arn:test",
4564 "CheckpointToken": "tok",
4565 "InitialExecutionState": { "NextMarker": "m-1" }
4566 });
4567 assert_eq!(
4568 inline_input_payload(&no_operations).unwrap(),
4569 client::PageInput::NoExecutionOperation
4570 );
4571 let later = envelope(serde_json::json!([
4573 { "Id": "s1", "Type": "STEP", "ExecutionDetails": { "InputPayload": "\"wrong\"" } },
4574 { "Id": "e", "Type": "EXECUTION", "ExecutionDetails": { "InputPayload": "\"right\"" } }
4575 ]));
4576 assert_eq!(
4577 inline_input_payload(&later).unwrap(),
4578 client::PageInput::Execution(Some("\"right\"".to_owned()))
4579 );
4580 let null = envelope(serde_json::json!([
4581 { "Id": "e", "Type": "EXECUTION", "ExecutionDetails": { "InputPayload": null } }
4582 ]));
4583 assert_eq!(
4584 inline_input_payload(&null).unwrap(),
4585 client::PageInput::Execution(None)
4586 );
4587 let number = envelope(serde_json::json!([
4588 { "Id": "e", "Type": "EXECUTION", "ExecutionDetails": { "InputPayload": 5 } }
4589 ]));
4590 let err = inline_input_payload(&number).unwrap_err();
4591 assert!(
4592 err.to_string().contains("InputPayload is not a string"),
4593 "got: {err}"
4594 );
4595 }
4596
4597 #[test]
4598 fn extract_customer_input_parse_failure_errors() {
4599 let err = extract_customer_input::<serde_json::Value>(Some("not json")).unwrap_err();
4600 assert!(
4601 err.to_string().contains("deserialize customer input"),
4602 "got: {err}"
4603 );
4604 }
4605
4606 struct PagedTransport {
4611 pages: std::collections::HashMap<String, GetStateOutput>,
4612 markers: std::sync::Mutex<Vec<Option<String>>>,
4613 }
4614
4615 impl PagedTransport {
4616 fn new(pages: Vec<(&str, Vec<aws_sdk_lambda::types::Operation>, Option<&str>)>) -> Self {
4617 Self {
4618 pages: pages
4619 .into_iter()
4620 .map(|(marker, ops, next)| {
4621 (
4622 marker.to_owned(),
4623 GetStateOutput::new(ops, next.map(str::to_owned)),
4624 )
4625 })
4626 .collect(),
4627 markers: std::sync::Mutex::new(Vec::new()),
4628 }
4629 }
4630
4631 fn markers(&self) -> Vec<Option<String>> {
4632 self.markers.lock().unwrap().clone()
4633 }
4634 }
4635
4636 impl ExecutionClient for PagedTransport {
4637 fn checkpoint(
4638 &self,
4639 _execution_arn: &str,
4640 _checkpoint_token: &str,
4641 _updates: Vec<aws_sdk_lambda::types::OperationUpdate>,
4642 ) -> std::pin::Pin<
4643 Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
4644 > {
4645 Box::pin(async {
4646 Ok(CheckpointOutput::new(
4647 Some("token-next".to_owned()),
4648 Vec::new(),
4649 None,
4650 ))
4651 })
4652 }
4653
4654 fn get_state(
4655 &self,
4656 _execution_arn: &str,
4657 _checkpoint_token: &str,
4658 marker: Option<&str>,
4659 ) -> std::pin::Pin<
4660 Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
4661 > {
4662 self.markers.lock().unwrap().push(marker.map(str::to_owned));
4663 let page = marker.and_then(|m| self.pages.get(m)).map_or_else(
4664 || GetStateOutput::new(Vec::new(), None),
4665 |p| GetStateOutput::new(p.operations.clone(), p.next_marker().map(str::to_owned)),
4666 );
4667 Box::pin(async move { Ok(page) })
4668 }
4669 }
4670
4671 async fn invoke_echo_over(
4674 payload: serde_json::Value,
4675 transport: std::sync::Arc<dyn ExecutionClient>,
4676 ) -> (
4677 Result<serde_json::Value, lambda_runtime::Error>,
4678 std::sync::Arc<std::sync::atomic::AtomicBool>,
4679 ) {
4680 let ran = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
4681 let ran_in_handler = std::sync::Arc::clone(&ran);
4682 let options = Options::builder()
4683 .execution_client(transport)
4684 .build()
4685 .expect("options build");
4686 let service = wrap(
4687 move |event: serde_json::Value, _ctx: DurableContext| {
4688 let ran = std::sync::Arc::clone(&ran_in_handler);
4689 async move {
4690 ran.store(true, std::sync::atomic::Ordering::SeqCst);
4691 Ok::<serde_json::Value, BoxError>(event)
4692 }
4693 },
4694 options,
4695 );
4696 let event = lambda_runtime::LambdaEvent::new(
4697 InvocationPayload(payload),
4698 lambda_runtime::Context::default(),
4699 );
4700 (service(event).await.map(|response| response.0), ran)
4701 }
4702
4703 fn bootstrap_envelope(state: &serde_json::Value) -> serde_json::Value {
4704 serde_json::json!({
4705 "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
4706 "CheckpointToken": "token-abc",
4707 "InitialExecutionState": state
4708 })
4709 }
4710
4711 fn assert_succeeded_with(response: &serde_json::Value, expected: &serde_json::Value) {
4712 assert_eq!(
4713 response.get("Status").and_then(serde_json::Value::as_str),
4714 Some("SUCCEEDED"),
4715 "unexpected response: {response}"
4716 );
4717 let result: serde_json::Value = serde_json::from_str(
4718 response
4719 .get("Result")
4720 .and_then(serde_json::Value::as_str)
4721 .expect("a Result string"),
4722 )
4723 .expect("the Result parses");
4724 assert_eq!(&result, expected, "unexpected response: {response}");
4725 }
4726
4727 #[tokio::test]
4730 async fn absent_input_payload_runs_on_empty_object() {
4731 for details in [
4732 serde_json::json!({}),
4733 serde_json::json!({ "InputPayload": null }),
4734 ] {
4735 let payload = bootstrap_envelope(&serde_json::json!({
4736 "Operations": [{
4737 "Id": "execution",
4738 "Type": "EXECUTION",
4739 "Status": "STARTED",
4740 "ExecutionDetails": details
4741 }]
4742 }));
4743 let transport = std::sync::Arc::new(PagedTransport::new(Vec::new()));
4744 let (response, _) =
4745 invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
4746 assert_succeeded_with(&response.expect("must succeed"), &serde_json::json!({}));
4747 assert!(transport.markers().is_empty(), "no marker, no get_state");
4748 }
4749 }
4750
4751 #[tokio::test]
4753 async fn input_payload_found_after_pagination() {
4754 let payload = bootstrap_envelope(&serde_json::json!({ "NextMarker": "m-1" }));
4755 let transport = std::sync::Arc::new(PagedTransport::new(vec![(
4756 "m-1",
4757 vec![make_test_execution_op("{\"count\":1}")],
4758 None,
4759 )]));
4760 let (response, _) = invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
4761 assert_succeeded_with(
4762 &response.expect("must succeed"),
4763 &serde_json::json!({ "count": 1 }),
4764 );
4765 assert_eq!(transport.markers(), vec![Some("m-1".to_owned())]);
4766 }
4767
4768 #[tokio::test]
4770 async fn paginated_input_matches_first_page() {
4771 let payload = bootstrap_envelope(&serde_json::json!({
4772 "Operations": [{
4773 "Id": "execution",
4774 "Type": "EXECUTION",
4775 "Status": "STARTED",
4776 "ExecutionDetails": { "InputPayload": "{\"a\":1}" }
4777 }],
4778 "NextMarker": "m-1"
4779 }));
4780 let transport = std::sync::Arc::new(PagedTransport::new(vec![(
4781 "m-1",
4782 vec![make_test_step_op("s1", "\"r1\"")],
4783 None,
4784 )]));
4785 let (response, _) = invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
4786 assert_succeeded_with(
4787 &response.expect("must succeed"),
4788 &serde_json::json!({ "a": 1 }),
4789 );
4790 assert_eq!(transport.markers(), vec![Some("m-1".to_owned())]);
4791 }
4792
4793 #[tokio::test]
4796 async fn malformed_input_payload_fails_the_invocation() {
4797 for (input, expected) in [
4798 (serde_json::json!("not json"), "deserialize customer input"),
4799 (serde_json::json!(5), "InputPayload is not a string"),
4800 ] {
4801 let payload = bootstrap_envelope(&serde_json::json!({
4802 "Operations": [{
4803 "Id": "execution",
4804 "Type": "EXECUTION",
4805 "Status": "STARTED",
4806 "ExecutionDetails": { "InputPayload": input }
4807 }]
4808 }));
4809 let transport = std::sync::Arc::new(PagedTransport::new(Vec::new()));
4810 let (response, ran) =
4811 invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
4812 let err = response.expect_err("must fail");
4813 assert!(err.to_string().contains(expected), "got: {err}");
4814 assert!(
4815 !ran.load(std::sync::atomic::Ordering::SeqCst),
4816 "the handler must not run"
4817 );
4818 assert!(transport.markers().is_empty(), "no marker, no get_state");
4819 }
4820 }
4821
4822 #[tokio::test]
4825 async fn missing_checkpoint_token_fails_before_input_default() {
4826 let payload = serde_json::json!({
4827 "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
4828 "InitialExecutionState": {
4829 "Operations": [{
4830 "Id": "execution",
4831 "Type": "EXECUTION",
4832 "Status": "STARTED",
4833 "ExecutionDetails": {}
4834 }]
4835 }
4836 });
4837 let transport = std::sync::Arc::new(PagedTransport::new(Vec::new()));
4838 let (response, ran) = invoke_echo_over(payload, transport).await;
4839 let err = response.expect_err("must fail");
4840 assert!(err.to_string().contains("CheckpointToken"), "got: {err}");
4841 assert!(
4842 !ran.load(std::sync::atomic::Ordering::SeqCst),
4843 "the handler must not run"
4844 );
4845 }
4846
4847 #[test]
4850 fn default_client_sets_connect_and_read_timeouts() {
4851 let config = SdkConfig::builder()
4852 .behavior_version(aws_config::BehaviorVersion::latest())
4853 .build();
4854 let client = default_lambda_client(&config);
4855 let timeouts = client
4856 .config()
4857 .timeout_config()
4858 .expect("the default client has a timeout config");
4859 assert_eq!(timeouts.connect_timeout(), Some(Duration::from_secs(5)));
4860 assert_eq!(timeouts.read_timeout(), Some(Duration::from_secs(50)));
4861 assert_eq!(
4862 timeouts.operation_attempt_timeout(),
4863 Some(Duration::from_secs(55))
4864 );
4865 }
4866
4867 #[test]
4868 fn supplied_config_keeps_its_timeouts() {
4869 use aws_sdk_lambda::config::timeout::TimeoutConfig;
4870 let config = SdkConfig::builder()
4871 .behavior_version(aws_config::BehaviorVersion::latest())
4872 .timeout_config(
4873 TimeoutConfig::builder()
4874 .connect_timeout(Duration::from_secs(1))
4875 .build(),
4876 )
4877 .build();
4878 let client = base_lambda_client_from_options(Some(config), None)
4879 .expect("sdk_config must yield a client");
4880 let timeouts = client.config().timeout_config().expect("a timeout config");
4881 assert_eq!(timeouts.connect_timeout(), Some(Duration::from_secs(1)));
4882 assert_eq!(timeouts.read_timeout(), None);
4883
4884 let conf = aws_sdk_lambda::config::Config::builder()
4885 .behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())
4886 .timeout_config(
4887 TimeoutConfig::builder()
4888 .read_timeout(Duration::from_secs(7))
4889 .build(),
4890 )
4891 .build();
4892 let supplied = aws_sdk_lambda::Client::from_conf(conf);
4893 let client = base_lambda_client_from_options(None, Some(supplied))
4894 .expect("lambda_client must be returned");
4895 let timeouts = client.config().timeout_config().expect("a timeout config");
4896 assert_eq!(timeouts.read_timeout(), Some(Duration::from_secs(7)));
4897 assert_eq!(timeouts.connect_timeout(), None);
4898 assert_eq!(timeouts.operation_attempt_timeout(), None);
4899 }
4900
4901 fn oversized_envelope() -> serde_json::Value {
4905 serde_json::json!({
4906 "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
4907 "CheckpointToken": "token-abc",
4908 "InitialExecutionState": { "Operations": [] }
4909 })
4910 }
4911
4912 async fn run_sized_result(
4916 n: usize,
4917 client: std::sync::Arc<client::InMemoryExecutionClient>,
4918 ) -> (
4919 Result<serde_json::Value, lambda_runtime::Error>,
4920 Vec<plugin::InvocationEndInfo>,
4921 ) {
4922 let hooks = Hooks::default();
4923 let options = Options::builder()
4924 .execution_client(client)
4925 .plugin(hooks.clone())
4926 .build()
4927 .expect("valid options");
4928 let service = wrap(
4930 move |_event: serde_json::Value, _ctx: DurableContext| async move {
4931 Ok::<String, BoxError>("a".repeat(n - 2))
4932 },
4933 options,
4934 );
4935 let response = service(lambda_runtime::LambdaEvent::new(
4936 InvocationPayload(oversized_envelope()),
4937 lambda_runtime::Context::default(),
4938 ))
4939 .await
4940 .map(|response| response.0);
4941 (response, hooks.ends())
4942 }
4943
4944 fn is_sized_result(serialized: &str, n: usize) -> bool {
4948 serialized.len() == n
4949 && serialized
4950 .strip_prefix('"')
4951 .and_then(|inner| inner.strip_suffix('"'))
4952 .is_some_and(|inner| inner.bytes().all(|b| b == b'a'))
4953 }
4954
4955 fn execution_updates(
4957 client: &client::InMemoryExecutionClient,
4958 ) -> Vec<aws_sdk_lambda::types::OperationUpdate> {
4959 client
4960 .recorded_updates()
4961 .into_iter()
4962 .filter(|update| update.r#type() == &aws_sdk_lambda::types::OperationType::Execution)
4963 .collect()
4964 }
4965
4966 #[tokio::test]
4970 async fn oversized_result_envelope_is_empty() {
4971 let n = LAMBDA_RESPONSE_SIZE_LIMIT + 1;
4972 assert_eq!(n, 6_291_407);
4973 let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
4974 let (response, ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
4975 let response = response.expect("an envelope");
4976 assert_eq!(
4977 response,
4978 serde_json::json!({ "Status": "SUCCEEDED", "Result": "" })
4979 );
4980 let updates = execution_updates(&client);
4981 let [update] = updates.as_slice() else {
4982 unreachable!("one EXECUTION update, got {}", updates.len());
4983 };
4984 assert_eq!(
4985 update.action(),
4986 &aws_sdk_lambda::types::OperationAction::Succeed
4987 );
4988 let payload = update.payload().unwrap_or_default();
4989 assert_eq!(payload.len(), n);
4990 assert!(
4991 is_sized_result(payload, n),
4992 "the payload is the serialized result"
4993 );
4994 let id = update.id();
4995 assert_eq!(id.len(), 64, "{id}");
4996 assert!(
4997 id.chars()
4998 .all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)),
4999 "{id}"
5000 );
5001 let [end] = ends.as_slice() else {
5002 unreachable!("one end, got {}", ends.len());
5003 };
5004 assert_eq!(end.status(), InvocationStatus::Succeeded);
5005 let recorded = end.execution_result().unwrap_or_default();
5006 assert_eq!(recorded.len(), n);
5007 assert!(
5008 is_sized_result(recorded, n),
5009 "the end info holds the full result"
5010 );
5011 }
5012
5013 #[tokio::test]
5018 async fn oversized_result_write_failures() {
5019 let n = LAMBDA_RESPONSE_SIZE_LIMIT + 1;
5020
5021 let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
5022 client.enqueue_checkpoint_response(client::TestResponse::NonRetryableError(
5023 "rejected".to_owned(),
5024 ));
5025 let (response, ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
5026 let response = response.expect("an envelope");
5027 assert_eq!(json_str(&response, "/Status"), Some("FAILED"), "{response}");
5028 assert_eq!(
5029 json_str(&response, "/Error/ErrorType"),
5030 Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
5031 );
5032 let message = json_str(&response, "/Error/ErrorMessage").unwrap_or_default();
5033 assert!(
5034 message.starts_with("[Unrecoverable Execution] Checkpoint failed: "),
5035 "{message}"
5036 );
5037 let [end] = ends.as_slice() else {
5038 unreachable!("one end, got {}", ends.len());
5039 };
5040 assert_eq!(end.status(), InvocationStatus::Failed);
5041 assert_eq!(end.execution_result(), None);
5042 assert_eq!(
5043 end.execution_error().and_then(WireError::error_type),
5044 Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
5045 );
5046 assert_eq!(
5047 end.execution_error().and_then(WireError::error_message),
5048 Some(message)
5049 );
5050
5051 let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
5052 client.enqueue_checkpoint_response(client::TestResponse::RetryableError(
5053 "throttled".to_owned(),
5054 ));
5055 let (response, ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
5056 let err = response.expect_err("the invocation fails");
5057 assert_eq!(
5058 err.to_string(),
5059 "[Unrecoverable Invocation] Checkpoint failed: throttled"
5060 );
5061 let [end] = ends.as_slice() else {
5062 unreachable!("one end, got {}", ends.len());
5063 };
5064 assert_eq!(end.status(), InvocationStatus::Retrying);
5065 assert_eq!(end.execution_result(), None);
5066 assert_eq!(
5067 end.execution_error().and_then(WireError::error_type),
5068 Some(error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE)
5069 );
5070 assert_eq!(
5071 end.execution_error().and_then(WireError::error_message),
5072 Some("[Unrecoverable Invocation] Checkpoint failed: throttled")
5073 );
5074 }
5075
5076 #[tokio::test]
5079 async fn oversized_result_after_withdrawal_is_pending() {
5080 let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
5081 let ctx = DurableContext::new_root_with_client_and_defaults(
5082 "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
5083 lambda_runtime::Context::default(),
5084 std::sync::Arc::new(engine::CheckpointLog::empty()),
5085 std::sync::Arc::clone(&client) as std::sync::Arc<dyn ExecutionClient>,
5086 "t0".to_owned(),
5087 Duration::ZERO,
5088 0,
5089 None,
5090 );
5091 ctx.coordinator()
5092 .withdraw(ctx.execution_arn(), "request", || {});
5093 let (response, end) = checkpoint_oversized_result(&ctx, "\"a\"".to_owned()).await;
5094 assert_eq!(
5095 response.expect("an envelope").0,
5096 serde_json::json!({ "Status": "PENDING" })
5097 );
5098 assert_eq!(end.status, InvocationStatus::Pending);
5099 assert_eq!(client.recorded_updates().len(), 0);
5100 }
5101
5102 #[tokio::test]
5105 async fn oversized_error_is_not_checkpointed() {
5106 let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
5107 let options = Options::builder()
5108 .execution_client(std::sync::Arc::clone(&client) as std::sync::Arc<dyn ExecutionClient>)
5109 .build()
5110 .expect("valid options");
5111 let service = wrap(
5113 |_event: serde_json::Value, _ctx: DurableContext| async {
5114 Err::<String, BoxError>("e".repeat(7 * 1024 * 1024).into())
5115 },
5116 options,
5117 );
5118 let response = service(lambda_runtime::LambdaEvent::new(
5119 InvocationPayload(oversized_envelope()),
5120 lambda_runtime::Context::default(),
5121 ))
5122 .await
5123 .expect("an envelope")
5124 .0;
5125 assert_eq!(json_str(&response, "/Status"), Some("FAILED"));
5126 assert_eq!(execution_updates(&client).len(), 0);
5127 }
5128
5129 #[tokio::test]
5131 async fn result_at_limit_stays_inline() {
5132 let n = LAMBDA_RESPONSE_SIZE_LIMIT;
5133 assert_eq!(n, 6_291_406);
5134 let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
5135 let (response, _ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
5136 let response = response.expect("an envelope");
5137 assert_eq!(json_str(&response, "/Status"), Some("SUCCEEDED"));
5138 let result = json_str(&response, "/Result").unwrap_or_default();
5139 assert_eq!(result.len(), n);
5140 assert!(is_sized_result(result, n), "the result is inline");
5141 assert_eq!(client.recorded_updates().len(), 0);
5142 }
5143}