1use std::pin::Pin;
6use std::sync::Arc;
7#[cfg(test)]
8use std::sync::Mutex;
9
10use aws_sdk_lambda::types::{Operation, OperationType, OperationUpdate};
11
12use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19#[non_exhaustive]
20pub enum ExecutionClientErrorScope {
21 Invocation,
23 Execution,
25}
26
27#[derive(Debug, Clone)]
50#[non_exhaustive]
51pub struct ExecutionClientError {
52 scope: ExecutionClientErrorScope,
53 message: String,
54 source: Option<Arc<dyn std::error::Error + Send + Sync>>,
55}
56
57impl std::fmt::Display for ExecutionClientError {
58 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59 write!(f, "durable: client: {}", self.message)
60 }
61}
62
63impl std::error::Error for ExecutionClientError {
64 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
65 self.source
66 .as_deref()
67 .map(|s| s as &(dyn std::error::Error + 'static))
68 }
69}
70
71impl ExecutionClientError {
72 #[must_use]
84 pub fn new(scope: ExecutionClientErrorScope, message: impl Into<String>) -> Self {
85 Self {
86 scope,
87 message: message.into(),
88 source: None,
89 }
90 }
91
92 #[must_use]
110 pub fn with_source(
111 scope: ExecutionClientErrorScope,
112 message: impl Into<String>,
113 source: impl Into<crate::BoxError>,
114 ) -> Self {
115 Self {
116 scope,
117 message: message.into(),
118 source: Some(Arc::from(source.into())),
119 }
120 }
121
122 pub fn scope(&self) -> ExecutionClientErrorScope {
124 self.scope
125 }
126
127 pub(crate) fn is_retryable(&self) -> bool {
131 self.scope == ExecutionClientErrorScope::Invocation
132 }
133
134 pub(crate) fn message(&self) -> &str {
137 &self.message
138 }
139
140 #[cfg(test)]
144 pub(crate) fn non_retryable(message: String) -> Self {
145 Self::new(ExecutionClientErrorScope::Execution, message)
146 }
147
148 pub(crate) fn new_non_retryable(message: &str) -> Self {
150 Self::new(ExecutionClientErrorScope::Execution, message)
151 }
152
153 #[cfg(any(test, feature = "test-util"))]
157 pub(crate) fn from_retryable(message: String) -> Self {
158 Self::new(ExecutionClientErrorScope::Invocation, message)
159 }
160}
161
162pub(crate) fn client_error_record_message(err: &ExecutionClientError) -> String {
176 use aws_sdk_lambda::error::ProvideErrorMetadata;
177 use std::error::Error as _;
178
179 if let Some(source) = err.source() {
180 if let Some(sdk) = source.downcast_ref::<CheckpointSdkError>()
181 && let Some(m) = sdk.message()
182 {
183 return m.to_owned();
184 }
185 if let Some(sdk) = source.downcast_ref::<GetStateSdkError>()
186 && let Some(m) = sdk.message()
187 {
188 return m.to_owned();
189 }
190 }
191 err.message().to_owned()
192}
193
194pub(crate) fn bootstrap_error_type(err: &ExecutionClientError) -> String {
202 use aws_sdk_lambda::error::ProvideErrorMetadata;
203 use std::error::Error as _;
204
205 err.source()
206 .and_then(|s| s.downcast_ref::<GetStateSdkError>())
207 .and_then(ProvideErrorMetadata::code)
208 .unwrap_or(crate::error::EXECUTION_CLIENT_ERROR_TYPE)
209 .to_owned()
210}
211
212type CheckpointSdkError = aws_sdk_lambda::error::SdkError<
214 aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError,
215 aws_sdk_lambda::config::http::HttpResponse,
216>;
217
218type GetStateSdkError = aws_sdk_lambda::error::SdkError<
220 aws_sdk_lambda::operation::get_durable_execution_state::GetDurableExecutionStateError,
221 aws_sdk_lambda::config::http::HttpResponse,
222>;
223
224#[derive(Debug, Clone)]
226#[non_exhaustive]
227pub struct CheckpointOutput {
228 pub(crate) checkpoint_token: Option<String>,
231 pub(crate) updated_operations: Vec<Operation>,
233 pub(crate) next_marker: Option<String>,
236}
237
238impl CheckpointOutput {
239 #[must_use]
245 pub fn new(
246 checkpoint_token: Option<String>,
247 updated_operations: Vec<Operation>,
248 next_marker: Option<String>,
249 ) -> Self {
250 Self {
251 checkpoint_token,
252 updated_operations,
253 next_marker,
254 }
255 }
256
257 pub fn checkpoint_token(&self) -> Option<&str> {
259 self.checkpoint_token.as_deref().filter(|t| !t.is_empty())
260 }
261}
262
263#[derive(Debug, Clone)]
265#[non_exhaustive]
266pub struct GetStateOutput {
267 pub(crate) operations: Vec<Operation>,
269 pub(crate) next_marker: Option<String>,
271}
272
273impl GetStateOutput {
274 #[must_use]
277 pub fn new(operations: Vec<Operation>, next_marker: Option<String>) -> Self {
278 Self {
279 operations,
280 next_marker,
281 }
282 }
283
284 pub fn next_marker(&self) -> Option<&str> {
286 self.next_marker.as_deref().filter(|m| !m.is_empty())
287 }
288}
289
290pub trait ExecutionClient: Send + Sync {
328 fn checkpoint(
330 &self,
331 execution_arn: &str,
332 checkpoint_token: &str,
333 updates: Vec<OperationUpdate>,
334 ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>;
335
336 fn get_state(
339 &self,
340 execution_arn: &str,
341 checkpoint_token: &str,
342 marker: Option<&str>,
343 ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>;
344}
345
346const GET_STATE_MAX_ITEMS: i32 = 1000;
348
349const USER_AGENT_TOKEN: &str =
356 concat!("aws-durable-execution-sdk-rust/", env!("CARGO_PKG_VERSION"));
357
358#[derive(Debug)]
369struct UserAgentTokenInterceptor;
370
371impl aws_sdk_lambda::config::Intercept for UserAgentTokenInterceptor {
372 fn name(&self) -> &'static str {
373 "UserAgentTokenInterceptor"
374 }
375
376 fn modify_before_transmit(
377 &self,
378 context: &mut aws_sdk_lambda::config::interceptors::BeforeTransmitInterceptorContextMut<'_>,
379 _runtime_components: &aws_sdk_lambda::config::RuntimeComponents,
380 _cfg: &mut aws_sdk_lambda::config::ConfigBag,
381 ) -> Result<(), aws_sdk_lambda::error::BoxError> {
382 let headers = context.request_mut().headers_mut();
383 let value = user_agent_with_token(headers.get("user-agent"));
384 headers.insert("user-agent", value);
388 Ok(())
389 }
390}
391
392fn user_agent_with_token(current: Option<&str>) -> String {
397 match current {
398 Some(current) => format!("{current} {USER_AGENT_TOKEN}"),
399 None => USER_AGENT_TOKEN.to_owned(),
400 }
401}
402
403#[derive(Clone)]
440#[non_exhaustive]
441pub struct LambdaExecutionClient {
442 client: aws_sdk_lambda::Client,
443}
444
445impl std::fmt::Debug for LambdaExecutionClient {
446 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
447 f.write_str("LambdaExecutionClient")
448 }
449}
450
451impl LambdaExecutionClient {
452 #[must_use]
454 pub fn new(client: crate::LambdaClient) -> Self {
455 Self { client }
456 }
457}
458
459impl ExecutionClient for LambdaExecutionClient {
460 fn checkpoint(
461 &self,
462 execution_arn: &str,
463 checkpoint_token: &str,
464 updates: Vec<OperationUpdate>,
465 ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
466 {
467 let arn = execution_arn.to_owned();
468 let token = checkpoint_token.to_owned();
469 Box::pin(async move {
470 let result = self
471 .client
472 .checkpoint_durable_execution()
473 .durable_execution_arn(&arn)
474 .checkpoint_token(&token)
475 .set_updates(Some(updates))
476 .customize()
477 .interceptor(UserAgentTokenInterceptor)
478 .send()
479 .await;
480
481 match result {
482 Ok(output) => {
483 let new_token = output.checkpoint_token.filter(|t| !t.is_empty());
484 let (updated_ops, next_marker) = match output.new_execution_state {
485 Some(state) => (
486 state.operations.unwrap_or_default(),
487 state.next_marker.filter(|m| !m.is_empty()),
488 ),
489 None => (Vec::new(), None),
490 };
491 Ok(CheckpointOutput::new(new_token, updated_ops, next_marker))
492 }
493 Err(err) => Err(classify_checkpoint_error(err)),
498 }
499 })
500 }
501
502 fn get_state(
506 &self,
507 execution_arn: &str,
508 checkpoint_token: &str,
509 marker: Option<&str>,
510 ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
511 {
512 let arn = execution_arn.to_owned();
513 let token = checkpoint_token.to_owned();
514 let marker = marker.map(str::to_owned);
515 Box::pin(async move {
516 let output = self
520 .client
521 .get_durable_execution_state()
522 .durable_execution_arn(&arn)
523 .checkpoint_token(&token)
524 .set_marker(marker)
525 .max_items(GET_STATE_MAX_ITEMS)
526 .customize()
527 .interceptor(UserAgentTokenInterceptor)
528 .send()
529 .await
530 .map_err(classify_get_state_error)?;
531
532 Ok(GetStateOutput::new(
533 output.operations,
534 output.next_marker.filter(|m| !m.is_empty()),
535 ))
536 })
537 }
538}
539
540pub(crate) async fn get_full_state(
544 client: &dyn ExecutionClient,
545 execution_arn: &str,
546 checkpoint_token: &str,
547) -> Result<Vec<Operation>, ExecutionClientError> {
548 let mut operations = Vec::new();
549 let mut marker: Option<String> = None;
550 loop {
551 let page = client
552 .get_state(execution_arn, checkpoint_token, marker.as_deref())
553 .await?;
554 let next = page.next_marker().map(str::to_owned);
555 operations.extend(page.operations);
556 match next {
557 Some(m) => marker = Some(m),
558 None => return Ok(operations),
559 }
560 }
561}
562
563const STALE_TOKEN_MESSAGE_PREFIX: &str = "Invalid checkpoint token";
570
571fn classify_checkpoint_error(err: CheckpointSdkError) -> ExecutionClientError {
616 let message = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(&err));
617 let scope = checkpoint_error_scope(&err);
618 ExecutionClientError::with_source(scope, message, err)
619}
620
621fn checkpoint_error_scope(err: &CheckpointSdkError) -> ExecutionClientErrorScope {
624 use ExecutionClientErrorScope::{Execution, Invocation};
625 use aws_sdk_lambda::error::SdkError;
626 use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError as E;
627
628 if let SdkError::ServiceError(service_err) = err {
629 match service_err.err() {
630 E::TooManyRequestsException(_) | E::ServiceException(_) => return Invocation,
632 E::InvalidParameterValueException(inner) => {
636 return if inner
637 .message()
638 .is_some_and(|m| m.starts_with(STALE_TOKEN_MESSAGE_PREFIX))
639 {
640 Invocation
641 } else {
642 Execution
643 };
644 }
645 E::KmsAccessDeniedException(_)
648 | E::KmsDisabledException(_)
649 | E::KmsInvalidStateException(_)
650 | E::KmsNotFoundException(_) => return Execution,
651 _ => {}
655 }
656 }
657
658 match err.raw_response().map(|r| r.status().as_u16()) {
662 Some(status) if (400..500).contains(&status) && status != 429 => Execution,
663 _ => Invocation,
664 }
665}
666
667fn classify_get_state_error(err: GetStateSdkError) -> ExecutionClientError {
678 use aws_sdk_lambda::operation::get_durable_execution_state::GetDurableExecutionStateError as E;
679
680 let message = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(&err));
681 let scope = match err.as_service_error() {
682 Some(
683 E::KmsAccessDeniedException(_)
684 | E::KmsDisabledException(_)
685 | E::KmsInvalidStateException(_)
686 | E::KmsNotFoundException(_),
687 ) => ExecutionClientErrorScope::Execution,
688 _ => ExecutionClientErrorScope::Invocation,
689 };
690 ExecutionClientError::with_source(scope, message, err)
691}
692
693#[cfg(test)]
698pub(crate) fn mock_aws_error_response(
699 status: u16,
700 code: &str,
701 message: &str,
702) -> aws_sdk_lambda::config::http::HttpResponse {
703 let body = serde_json::json!({ "Type": "User", "message": message }).to_string();
704 let mut response = aws_sdk_lambda::config::http::HttpResponse::new(
705 aws_smithy_runtime_api::http::StatusCode::try_from(status).expect("valid HTTP status"),
706 aws_smithy_types::body::SdkBody::from(body),
707 );
708 response
709 .headers_mut()
710 .insert("x-amzn-errortype", code.to_owned());
711 response
712 .headers_mut()
713 .insert("content-type", "application/json");
714 response
715}
716
717#[cfg(test)]
719#[derive(Debug, Clone)]
720pub(crate) enum TestResponse {
721 Success(Vec<Operation>),
723 SuccessPaginated(Vec<Operation>, String),
726 RetryableError(String),
728 NonRetryableError(String),
730 AcceptThenRetryableError(String),
733}
734
735#[cfg(test)]
740#[derive(Debug)]
741pub(crate) struct InMemoryExecutionClient {
742 state_operations: Mutex<Vec<Operation>>,
744 checkpoint_responses: Mutex<Vec<TestResponse>>,
746 pub(crate) checkpoint_call_count: Mutex<u32>,
748 pub(crate) get_state_call_count: Mutex<u32>,
750 get_state_failure: Mutex<Option<String>>,
754 token_counter: Mutex<u32>,
756 callback_counter: Mutex<u32>,
758 recorded_updates: Mutex<Vec<OperationUpdate>>,
760}
761
762#[cfg(test)]
763impl InMemoryExecutionClient {
764 pub(crate) fn new(state_operations: Vec<Operation>) -> Self {
766 Self {
767 state_operations: Mutex::new(state_operations),
768 checkpoint_responses: Mutex::new(Vec::new()),
769 checkpoint_call_count: Mutex::new(0),
770 get_state_call_count: Mutex::new(0),
771 get_state_failure: Mutex::new(None),
772 token_counter: Mutex::new(0),
773 callback_counter: Mutex::new(0),
774 recorded_updates: Mutex::new(Vec::new()),
775 }
776 }
777
778 pub(crate) fn enqueue_checkpoint_response(&self, response: TestResponse) {
780 let mut responses = self
781 .checkpoint_responses
782 .lock()
783 .unwrap_or_else(std::sync::PoisonError::into_inner);
784 responses.push(response);
785 }
786
787 pub(crate) fn fail_get_state(&self, message: &str) {
790 let mut failure = self
791 .get_state_failure
792 .lock()
793 .unwrap_or_else(std::sync::PoisonError::into_inner);
794 *failure = Some(message.to_owned());
795 }
796
797 pub(crate) fn recorded_updates(&self) -> Vec<OperationUpdate> {
799 self.recorded_updates
800 .lock()
801 .unwrap_or_else(std::sync::PoisonError::into_inner)
802 .clone()
803 }
804}
805
806#[cfg(test)]
807impl ExecutionClient for InMemoryExecutionClient {
808 fn checkpoint(
809 &self,
810 _execution_arn: &str,
811 _checkpoint_token: &str,
812 updates: Vec<OperationUpdate>,
813 ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
814 {
815 {
816 let mut count = self
817 .checkpoint_call_count
818 .lock()
819 .unwrap_or_else(std::sync::PoisonError::into_inner);
820 *count += 1;
821 }
822
823 let auto_ops: Vec<Operation> = updates
829 .iter()
830 .filter(|u| {
831 u.r#type() == &OperationType::Callback
832 && u.action() == &aws_sdk_lambda::types::OperationAction::Start
833 })
834 .map(|u| {
835 let mut counter = self
836 .callback_counter
837 .lock()
838 .unwrap_or_else(std::sync::PoisonError::into_inner);
839 *counter += 1;
840 Operation::builder()
841 .id(u.id())
842 .r#type(OperationType::Callback)
843 .status(aws_sdk_lambda::types::OperationStatus::Pending)
844 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
845 .callback_details(
846 aws_sdk_lambda::types::CallbackDetails::builder()
847 .callback_id(format!("in-mem-cb-{counter}"))
848 .build(),
849 )
850 .build()
851 .expect("all required Operation fields set")
852 })
853 .collect();
854
855 {
857 let mut recorded = self
858 .recorded_updates
859 .lock()
860 .unwrap_or_else(std::sync::PoisonError::into_inner);
861 recorded.extend(updates);
862 }
863
864 let response = {
865 let mut responses = self
866 .checkpoint_responses
867 .lock()
868 .unwrap_or_else(std::sync::PoisonError::into_inner);
869 if responses.is_empty() {
870 None
871 } else {
872 Some(responses.remove(0))
873 }
874 };
875
876 Box::pin(async move {
877 match response {
878 Some(
880 TestResponse::RetryableError(msg) | TestResponse::AcceptThenRetryableError(msg),
881 ) => Err(ExecutionClientError::from_retryable(msg)),
882 Some(TestResponse::NonRetryableError(msg)) => {
883 Err(ExecutionClientError::non_retryable(msg))
884 }
885 Some(TestResponse::Success(ops)) => {
886 let mut counter = self
887 .token_counter
888 .lock()
889 .unwrap_or_else(std::sync::PoisonError::into_inner);
890 *counter += 1;
891 Ok(CheckpointOutput {
892 checkpoint_token: Some(format!("token-{counter}")),
893 updated_operations: ops,
894 next_marker: None,
895 })
896 }
897 Some(TestResponse::SuccessPaginated(ops, marker)) => {
898 let mut counter = self
899 .token_counter
900 .lock()
901 .unwrap_or_else(std::sync::PoisonError::into_inner);
902 *counter += 1;
903 Ok(CheckpointOutput {
904 checkpoint_token: Some(format!("token-{counter}")),
905 updated_operations: ops,
906 next_marker: Some(marker),
907 })
908 }
909 None => {
910 let mut counter = self
911 .token_counter
912 .lock()
913 .unwrap_or_else(std::sync::PoisonError::into_inner);
914 *counter += 1;
915 Ok(CheckpointOutput {
916 checkpoint_token: Some(format!("token-{counter}")),
917 updated_operations: auto_ops,
918 next_marker: None,
919 })
920 }
921 }
922 })
923 }
924
925 fn get_state(
926 &self,
927 _execution_arn: &str,
928 _checkpoint_token: &str,
929 _marker: Option<&str>,
930 ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
931 {
932 {
933 let mut count = self
934 .get_state_call_count
935 .lock()
936 .unwrap_or_else(std::sync::PoisonError::into_inner);
937 *count += 1;
938 }
939
940 let failure = self
941 .get_state_failure
942 .lock()
943 .unwrap_or_else(std::sync::PoisonError::into_inner)
944 .clone();
945 if let Some(message) = failure {
946 return Box::pin(async move { Err(ExecutionClientError::non_retryable(message)) });
947 }
948
949 let ops = self
950 .state_operations
951 .lock()
952 .unwrap_or_else(std::sync::PoisonError::into_inner)
953 .clone();
954
955 Box::pin(async move { Ok(GetStateOutput::new(ops, None)) })
956 }
957}
958
959fn operation_to_record(op: &Operation) -> (String, CheckpointRecord) {
961 let attempt = op.step_details.as_ref().map_or(0, |s| {
962 #[expect(clippy::cast_sign_loss)] {
964 s.attempt.max(0) as u32
965 }
966 });
967 let record = CheckpointRecord {
968 id: op.id().to_owned(),
969 status: sdk_status_to_checkpoint_status(op.status()),
970 result: extract_result(op),
971 error_type: extract_error_type(op),
972 error_message: extract_error_message(op),
973 error_data: extract_wire_error_field(op, |e| e.error_data.clone()),
974 stack_trace: extract_wire_error_field(op, |e| {
975 (!e.stack_trace().is_empty()).then(|| e.stack_trace().to_vec())
976 }),
977 attempt,
978 invoke_result: op
982 .chained_invoke_details
983 .as_ref()
984 .and_then(|i| i.result.clone()),
985 invoke_error_type: op
986 .chained_invoke_details
987 .as_ref()
988 .and_then(|i| i.error.as_ref())
989 .and_then(|e| e.error_type.clone()),
990 invoke_error_message: op
991 .chained_invoke_details
992 .as_ref()
993 .and_then(|i| i.error.as_ref())
994 .and_then(|e| e.error_message.clone()),
995 invoke_error_data: op
996 .chained_invoke_details
997 .as_ref()
998 .and_then(|i| i.error.as_ref())
999 .and_then(|e| e.error_data.clone()),
1000 invoke_stack_trace: op
1001 .chained_invoke_details
1002 .as_ref()
1003 .and_then(|i| i.error.as_ref())
1004 .and_then(|e| (!e.stack_trace().is_empty()).then(|| e.stack_trace().to_vec())),
1005 replay_children: op
1008 .context_details
1009 .as_ref()
1010 .and_then(|c| c.replay_children)
1011 .unwrap_or(false),
1012 callback_id: op
1013 .callback_details
1014 .as_ref()
1015 .and_then(|cb| cb.callback_id.clone()),
1016 op_type: Some(operation_type_to_string(&op.r#type)),
1017 sub_type: op.sub_type.clone(),
1018 op_name: op.name.clone(),
1019 due_at: operation_due_at(op),
1020 parent_id: op.parent_id().map(str::to_owned),
1021 start_timestamp: crate::to_system_time(*op.start_timestamp()),
1022 end_timestamp: op.end_timestamp().and_then(|t| crate::to_system_time(*t)),
1023 };
1024 (op.id().to_owned(), record)
1025}
1026
1027fn operation_due_at(op: &Operation) -> Option<std::time::SystemTime> {
1031 let timestamp = match op.r#type() {
1032 OperationType::Wait => op
1033 .wait_details
1034 .as_ref()
1035 .and_then(|d| d.scheduled_end_timestamp),
1036 OperationType::Step => op
1037 .step_details
1038 .as_ref()
1039 .and_then(|d| d.next_attempt_timestamp),
1040 _ => None,
1041 }?;
1042 crate::to_system_time(timestamp)
1043}
1044
1045fn operation_type_to_string(op_type: &OperationType) -> String {
1048 op_type.as_str().to_owned()
1049}
1050
1051pub(crate) fn execution_info_from_operation(
1054 op: &Operation,
1055) -> Option<crate::plugin::OperationInfo> {
1056 crate::plugin::OperationInfo::execution(
1057 op.id(),
1058 op.r#type().as_str(),
1059 &sdk_status_to_checkpoint_status(op.status()),
1060 op.parent_id().map(str::to_owned),
1061 crate::to_system_time(*op.start_timestamp()),
1062 op.end_timestamp().and_then(|t| crate::to_system_time(*t)),
1063 )
1064}
1065
1066#[cfg(test)]
1072pub(crate) fn operations_to_checkpoint_log(operations: &[Operation]) -> CheckpointLog {
1073 let records: Vec<(String, CheckpointRecord)> = operations
1074 .iter()
1075 .filter(|op| op.r#type != OperationType::Execution)
1078 .map(operation_to_record)
1079 .collect();
1080 CheckpointLog::from_records(records)
1081}
1082
1083pub(crate) fn merge_operations_into_log(
1095 log: &CheckpointLog,
1096 operations: &[Operation],
1097) -> Vec<String> {
1098 let mut changed = Vec::new();
1099 for op in operations {
1100 if op.r#type == OperationType::Execution {
1101 continue;
1102 }
1103 let (wire_id, record) = operation_to_record(op);
1104 let status = record.status.clone();
1105 if log.insert(wire_id.clone(), record).as_ref() != Some(&status) {
1106 changed.push(wire_id);
1107 }
1108 }
1109 changed
1110}
1111
1112#[derive(Debug)]
1115pub(crate) struct BootstrapState {
1116 pub(crate) log: CheckpointLog,
1118 pub(crate) input_payload: Option<String>,
1121 pub(crate) execution: Option<crate::plugin::OperationInfo>,
1124}
1125
1126pub(crate) async fn resolve_bootstrap_log(
1147 client: &dyn ExecutionClient,
1148 execution_arn: &str,
1149 checkpoint_token: &str,
1150 first_page: BootstrapState,
1151 next_marker: Option<&str>,
1152) -> Result<BootstrapState, ExecutionClientError> {
1153 let mut state = first_page;
1154 let mut marker = next_marker.map(str::to_owned);
1155 while let Some(current) = marker {
1156 let page = client
1157 .get_state(execution_arn, checkpoint_token, Some(¤t))
1158 .await?;
1159 marker = page.next_marker().map(str::to_owned);
1160 if state.execution.is_none()
1163 && let Some(op) = page
1164 .operations
1165 .iter()
1166 .find(|op| op.r#type == OperationType::Execution)
1167 {
1168 state.input_payload = op
1169 .execution_details
1170 .as_ref()
1171 .and_then(|d| d.input_payload.clone());
1172 state.execution = execution_info_from_operation(op);
1173 }
1174 for op in page
1175 .operations
1176 .iter()
1177 .filter(|op| op.r#type != OperationType::Execution)
1178 {
1179 let (wire_id, record) = operation_to_record(op);
1180 state.log.insert(wire_id, record);
1181 }
1182 }
1183 Ok(state)
1184}
1185
1186#[derive(Debug, Clone, PartialEq, Eq)]
1193pub(crate) enum PageInput {
1194 NoExecutionOperation,
1196 Execution(Option<String>),
1199}
1200
1201impl PageInput {
1202 pub(crate) fn into_payload(self) -> Option<String> {
1205 match self {
1206 Self::NoExecutionOperation => None,
1207 Self::Execution(payload) => payload,
1208 }
1209 }
1210}
1211
1212fn sdk_status_to_checkpoint_status(
1222 status: &aws_sdk_lambda::types::OperationStatus,
1223) -> CheckpointStatus {
1224 match status {
1225 aws_sdk_lambda::types::OperationStatus::Started => CheckpointStatus::Started,
1226 aws_sdk_lambda::types::OperationStatus::Pending => CheckpointStatus::Pending,
1227 aws_sdk_lambda::types::OperationStatus::Ready => CheckpointStatus::Ready,
1228 aws_sdk_lambda::types::OperationStatus::Succeeded => CheckpointStatus::Succeeded,
1229 aws_sdk_lambda::types::OperationStatus::Failed => CheckpointStatus::Failed,
1230 aws_sdk_lambda::types::OperationStatus::Cancelled => CheckpointStatus::Cancelled,
1231 aws_sdk_lambda::types::OperationStatus::TimedOut => CheckpointStatus::TimedOut,
1232 aws_sdk_lambda::types::OperationStatus::Stopped => CheckpointStatus::Stopped,
1233 other => CheckpointStatus::Unknown(other.as_str().to_owned()),
1234 }
1235}
1236
1237fn extract_result(op: &Operation) -> Option<String> {
1243 op.step_details
1244 .as_ref()
1245 .and_then(|s| s.result.clone())
1246 .or_else(|| op.context_details.as_ref().and_then(|c| c.result.clone()))
1247 .or_else(|| {
1248 op.callback_details
1249 .as_ref()
1250 .and_then(|cb| cb.result.clone())
1251 })
1252}
1253
1254fn extract_error_type(op: &Operation) -> Option<String> {
1260 op.step_details
1261 .as_ref()
1262 .and_then(|s| s.error.as_ref())
1263 .and_then(|e| e.error_type.clone())
1264 .or_else(|| {
1265 op.context_details
1266 .as_ref()
1267 .and_then(|c| c.error.as_ref())
1268 .and_then(|e| e.error_type.clone())
1269 })
1270 .or_else(|| {
1271 op.callback_details
1272 .as_ref()
1273 .and_then(|cb| cb.error.as_ref())
1274 .and_then(|e| e.error_type.clone())
1275 })
1276}
1277
1278fn extract_error_message(op: &Operation) -> Option<String> {
1284 op.step_details
1285 .as_ref()
1286 .and_then(|s| s.error.as_ref())
1287 .and_then(|e| e.error_message.clone())
1288 .or_else(|| {
1289 op.context_details
1290 .as_ref()
1291 .and_then(|c| c.error.as_ref())
1292 .and_then(|e| e.error_message.clone())
1293 })
1294 .or_else(|| {
1295 op.callback_details
1296 .as_ref()
1297 .and_then(|cb| cb.error.as_ref())
1298 .and_then(|e| e.error_message.clone())
1299 })
1300}
1301
1302fn extract_wire_error_field<T>(
1306 op: &Operation,
1307 read: impl Fn(&aws_sdk_lambda::types::ErrorObject) -> Option<T>,
1308) -> Option<T> {
1309 op.step_details
1310 .as_ref()
1311 .and_then(|s| s.error.as_ref())
1312 .and_then(&read)
1313 .or_else(|| {
1314 op.context_details
1315 .as_ref()
1316 .and_then(|c| c.error.as_ref())
1317 .and_then(&read)
1318 })
1319 .or_else(|| {
1320 op.callback_details
1321 .as_ref()
1322 .and_then(|cb| cb.error.as_ref())
1323 .and_then(&read)
1324 })
1325}
1326
1327#[cfg(test)]
1328mod tests {
1329 use super::*;
1330
1331 #[test]
1333 fn sdk_operation_parent_and_timestamps() {
1334 use std::time::{Duration, UNIX_EPOCH};
1335 let with_all = Operation::builder()
1336 .id("s")
1337 .r#type(OperationType::Step)
1338 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
1339 .parent_id("p")
1340 .start_timestamp(aws_smithy_types::DateTime::from_secs(10))
1341 .end_timestamp(aws_smithy_types::DateTime::from_secs(12))
1342 .build()
1343 .expect("all required fields set");
1344 let (_, record) = operation_to_record(&with_all);
1345 assert_eq!(record.parent_id.as_deref(), Some("p"));
1346 assert_eq!(
1347 record.start_timestamp,
1348 Some(UNIX_EPOCH + Duration::from_secs(10))
1349 );
1350 assert_eq!(
1351 record.end_timestamp,
1352 Some(UNIX_EPOCH + Duration::from_secs(12))
1353 );
1354
1355 let bare = Operation::builder()
1356 .id("s")
1357 .r#type(OperationType::Step)
1358 .status(aws_sdk_lambda::types::OperationStatus::Started)
1359 .start_timestamp(aws_smithy_types::DateTime::from_secs(10))
1360 .build()
1361 .expect("all required fields set");
1362 let (_, record) = operation_to_record(&bare);
1363 assert_eq!(record.parent_id, None);
1364 assert_eq!(record.end_timestamp, None);
1365 }
1366
1367 #[test]
1370 fn unknown_type_and_status_keep_wire_tokens_on_get_state_path() {
1371 let op = Operation::builder()
1372 .id("f")
1373 .r#type(OperationType::from("FUTURE_OP"))
1374 .status(aws_sdk_lambda::types::OperationStatus::from(
1375 "FUTURE_STATUS",
1376 ))
1377 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
1378 .build()
1379 .expect("all required fields set");
1380 let (id, record) = operation_to_record(&op);
1381 let info = crate::plugin::OperationInfo::from_record(&id, &record);
1382 assert_eq!(info.kind(), None);
1383 assert_eq!(info.status(), None);
1384 assert_eq!(info.type_str(), Some("FUTURE_OP"));
1385 assert_eq!(info.status_str(), Some("FUTURE_STATUS"));
1386 }
1387
1388 #[test]
1390 fn operation_type_to_string_returns_wire_tokens() {
1391 for (op_type, token) in [
1392 (OperationType::Callback, "CALLBACK"),
1393 (OperationType::ChainedInvoke, "CHAINED_INVOKE"),
1394 (OperationType::Context, "CONTEXT"),
1395 (OperationType::Execution, "EXECUTION"),
1396 (OperationType::Step, "STEP"),
1397 (OperationType::Wait, "WAIT"),
1398 ] {
1399 assert_eq!(operation_type_to_string(&op_type), token);
1400 }
1401 }
1402
1403 #[test]
1406 fn execution_info_from_operation_reads_execution_only() {
1407 use std::time::{Duration, UNIX_EPOCH};
1408 let execution = Operation::builder()
1409 .id("exec")
1410 .r#type(OperationType::Execution)
1411 .status(aws_sdk_lambda::types::OperationStatus::Started)
1412 .start_timestamp(aws_smithy_types::DateTime::from_secs(3))
1413 .end_timestamp(aws_smithy_types::DateTime::from_secs(4))
1414 .build()
1415 .expect("all required fields set");
1416 let info = execution_info_from_operation(&execution).expect("an EXECUTION operation");
1417 assert_eq!(info.id(), "exec");
1418 assert_eq!(info.type_str(), Some("EXECUTION"));
1419 assert_eq!(info.status_str(), Some("STARTED"));
1420 assert_eq!(info.parent_id(), None);
1421 assert_eq!(
1422 info.start_timestamp(),
1423 Some(UNIX_EPOCH + Duration::from_secs(3))
1424 );
1425 assert_eq!(
1426 info.end_timestamp(),
1427 Some(UNIX_EPOCH + Duration::from_secs(4))
1428 );
1429 let step = Operation::builder()
1430 .id("s")
1431 .r#type(OperationType::Step)
1432 .status(aws_sdk_lambda::types::OperationStatus::Started)
1433 .start_timestamp(aws_smithy_types::DateTime::from_secs(3))
1434 .build()
1435 .expect("all required fields set");
1436 assert!(execution_info_from_operation(&step).is_none());
1437 }
1438
1439 #[test]
1440 fn due_at_from_wait_and_step_details() {
1441 let op = |r#type: OperationType| {
1442 Operation::builder()
1443 .id("op")
1444 .r#type(r#type)
1445 .status(aws_sdk_lambda::types::OperationStatus::Started)
1446 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
1447 };
1448 let wait = op(OperationType::Wait)
1449 .wait_details(
1450 aws_sdk_lambda::types::WaitDetails::builder()
1451 .scheduled_end_timestamp(aws_smithy_types::DateTime::from_secs(100))
1452 .build(),
1453 )
1454 .build()
1455 .expect("all required fields set");
1456 let step = op(OperationType::Step)
1457 .step_details(
1458 aws_sdk_lambda::types::StepDetails::builder()
1459 .next_attempt_timestamp(aws_smithy_types::DateTime::from_secs(200))
1460 .build(),
1461 )
1462 .build()
1463 .expect("all required fields set");
1464 let neither = op(OperationType::Wait)
1465 .build()
1466 .expect("all required fields set");
1467 let callback = op(OperationType::Callback)
1469 .wait_details(
1470 aws_sdk_lambda::types::WaitDetails::builder()
1471 .scheduled_end_timestamp(aws_smithy_types::DateTime::from_secs(300))
1472 .build(),
1473 )
1474 .build()
1475 .expect("all required fields set");
1476 let epoch = std::time::UNIX_EPOCH;
1477 assert_eq!(
1478 operation_to_record(&wait).1.due_at,
1479 Some(epoch + std::time::Duration::from_secs(100))
1480 );
1481 assert_eq!(
1482 operation_to_record(&step).1.due_at,
1483 Some(epoch + std::time::Duration::from_secs(200))
1484 );
1485 assert_eq!(operation_to_record(&neither).1.due_at, None);
1486 assert_eq!(operation_to_record(&callback).1.due_at, None);
1487 }
1488
1489 use aws_sdk_lambda::operation::checkpoint_durable_execution::{
1497 CheckpointDurableExecutionError, CheckpointDurableExecutionOutput,
1498 };
1499 use aws_sdk_lambda::operation::get_durable_execution_state::{
1500 GetDurableExecutionStateError, GetDurableExecutionStateOutput,
1501 };
1502 use aws_sdk_lambda::types::{CheckpointUpdatedExecutionState, OperationAction};
1503 use aws_smithy_mocks::{RuleMode, mock, mock_client};
1504
1505 fn make_update(id: &str, action: OperationAction) -> OperationUpdate {
1507 OperationUpdate::builder()
1508 .id(id)
1509 .r#type(OperationType::Step)
1510 .action(action)
1511 .build()
1512 .expect("valid OperationUpdate")
1513 }
1514
1515 #[tokio::test]
1519 async fn checkpoint_maps_request_and_response() {
1520 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1521 .match_requests(|req| {
1522 req.durable_execution_arn() == Some("arn:aws:lambda:us-east-1:1:function:f")
1523 && req.checkpoint_token() == Some("tok-in")
1524 && req.updates().len() == 2
1525 && req.updates().first().map(OperationUpdate::id) == Some("op-1")
1526 && req.updates().get(1).map(OperationUpdate::id) == Some("op-2")
1527 })
1528 .then_output(|| {
1529 CheckpointDurableExecutionOutput::builder()
1530 .checkpoint_token("tok-out")
1531 .new_execution_state(
1532 CheckpointUpdatedExecutionState::builder()
1533 .operations(make_step_op("op-1", r#""r1""#))
1534 .next_marker("more-pages")
1535 .build(),
1536 )
1537 .build()
1538 });
1539 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1540 let client = LambdaExecutionClient::new(sdk_client);
1541
1542 let updates = vec![
1543 make_update("op-1", OperationAction::Start),
1544 make_update("op-2", OperationAction::Succeed),
1545 ];
1546 let output = client
1547 .checkpoint("arn:aws:lambda:us-east-1:1:function:f", "tok-in", updates)
1548 .await
1549 .expect("mocked checkpoint succeeds");
1550
1551 assert_eq!(output.checkpoint_token(), Some("tok-out"));
1552 assert_eq!(output.updated_operations.len(), 1);
1553 assert_eq!(
1554 output.updated_operations.first().map(Operation::id),
1555 Some("op-1")
1556 );
1557 assert_eq!(output.next_marker.as_deref(), Some("more-pages"));
1558 assert_eq!(rule.num_calls(), 1);
1559 }
1560
1561 const FIRST_ROOT_WIRE_ID: &str =
1563 "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b";
1564
1565 async fn invoke_one_step(sdk_client: aws_sdk_lambda::Client) -> serde_json::Value {
1572 let service = crate::wrap(
1573 |_event: serde_json::Value, ctx: crate::DurableContext| async move {
1574 let value: u32 = ctx.step(|_| async { Ok(7_u32) }).name("one").await?;
1575 Ok::<u32, crate::BoxError>(value)
1576 },
1577 crate::Options::builder()
1578 .execution_client(Arc::new(LambdaExecutionClient::new(sdk_client)))
1579 .build()
1580 .expect("options build"),
1581 );
1582 let event = lambda_runtime::LambdaEvent::new(
1583 crate::InvocationPayload(serde_json::json!({
1584 "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
1585 "CheckpointToken": "token-abc",
1586 "InitialExecutionState": {
1587 "Operations": [
1588 {
1589 "Id": "root",
1590 "Type": "Execution",
1591 "Status": "STARTED",
1592 "ExecutionDetails": { "InputPayload": "{}" }
1593 }
1594 ]
1595 }
1596 })),
1597 lambda_runtime::Context::default(),
1598 );
1599 service(event)
1600 .await
1601 .map(|response| response.0)
1602 .expect("a tokenless response answers PENDING, not an invocation error")
1603 }
1604
1605 #[tokio::test]
1615 async fn checkpoint_output_without_token_answers_pending_unless_execution_update() {
1616 use std::sync::{Arc, Mutex as StdMutex};
1617
1618 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1619 .then_output(|| CheckpointDurableExecutionOutput::builder().build());
1620 let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
1621 let output = client
1622 .checkpoint("arn:test", "tok", Vec::new())
1623 .await
1624 .expect("a tokenless response is Ok");
1625 assert_eq!(output.checkpoint_token(), None);
1626
1627 let requests: Arc<StdMutex<Vec<Vec<OperationUpdate>>>> = Arc::default();
1629 let seen = Arc::clone(&requests);
1630 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1631 .match_requests(move |req| {
1632 seen.lock()
1633 .unwrap_or_else(std::sync::PoisonError::into_inner)
1634 .push(req.updates().to_vec());
1635 true
1636 })
1637 .then_output(|| CheckpointDurableExecutionOutput::builder().build());
1638 let response =
1639 invoke_one_step(mock_client!(aws_sdk_lambda, RuleMode::MatchAny, [&rule])).await;
1640 assert_eq!(response, serde_json::json!({ "Status": "PENDING" }));
1641 let requests = requests
1642 .lock()
1643 .unwrap_or_else(std::sync::PoisonError::into_inner)
1644 .clone();
1645 assert_eq!(requests.len(), 1, "{requests:?}");
1646 let first = requests.first().expect("one request");
1647 assert_eq!(first.len(), 1, "{first:?}");
1648 let start = first.first().expect("the step's START");
1649 assert_eq!(start.id(), FIRST_ROOT_WIRE_ID);
1650 assert_eq!(start.action(), &OperationAction::Start);
1651 assert!(
1652 requests
1653 .iter()
1654 .flatten()
1655 .all(|u| u.action() != &OperationAction::Fail),
1656 "no terminal FAIL is sent: {requests:?}"
1657 );
1658
1659 let (messages, _guard) = crate::driver::test_support::EventMessages::install();
1660 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1661 .then_output(|| CheckpointDurableExecutionOutput::builder().build());
1662 let ctx = crate::DurableContext::new_root_with_client(
1663 "arn:test".to_owned(),
1664 lambda_runtime::Context::default(),
1665 Arc::new(CheckpointLog::empty()),
1666 Arc::new(LambdaExecutionClient::new(mock_client!(
1667 aws_sdk_lambda,
1668 [&rule]
1669 ))),
1670 "tok".to_owned(),
1671 );
1672 let succeed = OperationUpdate::builder()
1673 .id("root")
1674 .r#type(OperationType::Execution)
1675 .action(OperationAction::Succeed)
1676 .payload("1")
1677 .build()
1678 .expect("valid update");
1679 ctx.checkpoint_updates_direct(vec![succeed])
1680 .await
1681 .expect("an EXECUTION update is exempt from withdrawal");
1682 assert!(!ctx.coordinator().withdrawn());
1683 assert_eq!(
1684 ctx.coordinator().phase(),
1685 crate::coordinator::Phase::Running
1686 );
1687 assert!(
1688 messages.at(tracing::Level::WARN).is_empty(),
1689 "{:?}",
1690 messages.at(tracing::Level::WARN)
1691 );
1692 }
1693
1694 #[test]
1697 fn execution_client_error_new_keeps_scope_and_message() {
1698 use std::error::Error as _;
1699
1700 let err = ExecutionClientError::new(ExecutionClientErrorScope::Execution, "x");
1701 assert_eq!(err.scope(), ExecutionClientErrorScope::Execution);
1702 assert!(!err.is_retryable());
1703 assert!(err.source().is_none());
1704 assert_eq!(err.to_string(), "durable: client: x");
1705
1706 let err = ExecutionClientError::new(ExecutionClientErrorScope::Invocation, "x");
1707 assert_eq!(err.scope(), ExecutionClientErrorScope::Invocation);
1708 assert!(err.is_retryable());
1709 }
1710
1711 #[tokio::test]
1714 async fn lambda_transport_error_source_downcasts_to_sdk_error() {
1715 use std::error::Error as _;
1716
1717 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1718 .sequence()
1719 .http_status(403, None)
1720 .build();
1721 let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
1722 let err = client
1723 .checkpoint("arn:test", "tok", Vec::new())
1724 .await
1725 .expect_err("a 403 fails the call");
1726
1727 let source = err.source().expect("the transport keeps the SdkError");
1728 let sdk = source
1729 .downcast_ref::<CheckpointSdkError>()
1730 .expect("the source is the final SdkError");
1731 let text = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(sdk));
1732 assert_eq!(err.message(), text);
1733 assert_eq!(format!("{err:#}"), format!("durable: client: {text}"));
1734 assert_eq!(err.scope(), ExecutionClientErrorScope::Execution);
1735 }
1736
1737 #[tokio::test]
1743 async fn checkpoint_final_error_maps_to_non_retryable_client_error() {
1744 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_error(|| {
1745 CheckpointDurableExecutionError::InvalidParameterValueException(
1746 aws_sdk_lambda::types::error::InvalidParameterValueException::builder().build(),
1747 )
1748 });
1749 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1750 let client = LambdaExecutionClient::new(sdk_client);
1751
1752 let err = client
1753 .checkpoint("arn:test", "tok", Vec::new())
1754 .await
1755 .expect_err("modeled error must fail the call");
1756 assert!(!err.is_retryable());
1757 assert_eq!(rule.num_calls(), 1, "a client fault is not retried");
1758 }
1759
1760 #[tokio::test]
1766 async fn checkpoint_stale_token_error_maps_to_retryable_client_error() {
1767 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_error(|| {
1768 CheckpointDurableExecutionError::InvalidParameterValueException(
1769 aws_sdk_lambda::types::error::InvalidParameterValueException::builder()
1770 .message("Invalid checkpoint token: token has been superseded")
1771 .build(),
1772 )
1773 });
1774 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1775 let client = LambdaExecutionClient::new(sdk_client);
1776
1777 let err = client
1778 .checkpoint("arn:test", "tok", Vec::new())
1779 .await
1780 .expect_err("stale checkpoint token must fail the call");
1781 assert!(err.is_retryable());
1782 assert_eq!(rule.num_calls(), 1, "a client fault is not retried");
1783 }
1784
1785 #[tokio::test]
1791 async fn checkpoint_retry_is_the_sdk_standard_retry_alone() {
1792 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1793 .sequence()
1794 .http_status(503, None)
1795 .repeatedly()
1796 .build();
1797 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1798 let client = LambdaExecutionClient::new(sdk_client);
1799
1800 let err = client
1801 .checkpoint("arn:test", "tok", Vec::new())
1802 .await
1803 .expect_err("persistent 503 must exhaust retries");
1804 assert!(err.is_retryable());
1805 assert_eq!(
1806 rule.num_calls(),
1807 3,
1808 "attempts must equal the SDK standard-retry budget — no outer loop multiplying them"
1809 );
1810 }
1811
1812 #[tokio::test]
1815 async fn get_state_returns_one_page_with_marker() {
1816 let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
1817 .match_requests(|req| {
1818 req.durable_execution_arn() == Some("arn:test")
1819 && req.checkpoint_token() == Some("tok")
1820 && req.marker() == Some("m-1")
1821 && req.max_items() == Some(1000)
1822 })
1823 .then_output(|| {
1824 GetDurableExecutionStateOutput::builder()
1825 .operations(make_step_op("op-1", r#""p1""#))
1826 .operations(make_step_op("op-2", r#""p2""#))
1827 .next_marker("m-2")
1828 .build()
1829 .expect("operations is set")
1830 });
1831 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1832 let client = LambdaExecutionClient::new(sdk_client);
1833
1834 let page = client
1835 .get_state("arn:test", "tok", Some("m-1"))
1836 .await
1837 .expect("get_state succeeds");
1838
1839 assert_eq!(rule.num_calls(), 1);
1840 assert_eq!(page.operations.len(), 2);
1841 assert_eq!(page.operations.first().map(Operation::id), Some("op-1"));
1842 assert_eq!(page.operations.get(1).map(Operation::id), Some("op-2"));
1843 assert_eq!(page.next_marker(), Some("m-2"));
1844 }
1845
1846 struct ThreePageClient {
1849 markers: Mutex<Vec<Option<String>>>,
1850 }
1851
1852 impl ExecutionClient for ThreePageClient {
1853 fn checkpoint(
1854 &self,
1855 _execution_arn: &str,
1856 _checkpoint_token: &str,
1857 _updates: Vec<OperationUpdate>,
1858 ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
1859 {
1860 Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
1861 }
1862
1863 fn get_state(
1864 &self,
1865 _execution_arn: &str,
1866 _checkpoint_token: &str,
1867 marker: Option<&str>,
1868 ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
1869 {
1870 self.markers
1871 .lock()
1872 .unwrap_or_else(std::sync::PoisonError::into_inner)
1873 .push(marker.map(str::to_owned));
1874 let page = match marker {
1875 None => GetStateOutput::new(vec![make_step_op("a", "1")], Some("p-2".to_owned())),
1876 Some("p-2") => {
1877 GetStateOutput::new(vec![make_step_op("b", "2")], Some("p-3".to_owned()))
1878 }
1879 Some(_) => GetStateOutput::new(vec![make_step_op("c", "3")], None),
1880 };
1881 Box::pin(async move { Ok(page) })
1882 }
1883 }
1884
1885 #[tokio::test]
1888 async fn sdk_loops_get_state_until_marker_empty() {
1889 let client = ThreePageClient {
1890 markers: Mutex::new(Vec::new()),
1891 };
1892 let operations = get_full_state(&client, "arn:test", "tok")
1893 .await
1894 .expect("whole state");
1895
1896 let ids: Vec<&str> = operations.iter().map(Operation::id).collect();
1897 assert_eq!(ids, ["a", "b", "c"]);
1898 let markers = client
1899 .markers
1900 .lock()
1901 .unwrap_or_else(std::sync::PoisonError::into_inner)
1902 .clone();
1903 assert_eq!(
1904 markers,
1905 [None, Some("p-2".to_owned()), Some("p-3".to_owned())]
1906 );
1907 }
1908
1909 struct InlineContinuationClient {
1914 markers: Mutex<Vec<Option<String>>>,
1915 }
1916
1917 impl ExecutionClient for InlineContinuationClient {
1918 fn checkpoint(
1919 &self,
1920 _execution_arn: &str,
1921 _checkpoint_token: &str,
1922 _updates: Vec<OperationUpdate>,
1923 ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
1924 {
1925 Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
1926 }
1927
1928 fn get_state(
1929 &self,
1930 _execution_arn: &str,
1931 _checkpoint_token: &str,
1932 marker: Option<&str>,
1933 ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
1934 {
1935 self.markers
1936 .lock()
1937 .unwrap_or_else(std::sync::PoisonError::into_inner)
1938 .push(marker.map(str::to_owned));
1939 let page = match marker {
1940 Some("m-1") => {
1941 GetStateOutput::new(vec![make_step_op("s2", "2")], Some("m-2".to_owned()))
1942 }
1943 Some("m-2") => GetStateOutput::new(vec![make_step_op("s3", "3")], None),
1944 _ => GetStateOutput::new(vec![make_step_op("s0", "0")], None),
1945 };
1946 Box::pin(async move { Ok(page) })
1947 }
1948 }
1949
1950 #[tokio::test]
1954 async fn bootstrap_keeps_inline_page_and_continues_from_marker() {
1955 let client = InlineContinuationClient {
1956 markers: Mutex::new(Vec::new()),
1957 };
1958 let execution = Operation::builder()
1959 .id("execution")
1960 .r#type(OperationType::Execution)
1961 .status(aws_sdk_lambda::types::OperationStatus::Started)
1962 .start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
1963 .execution_details(
1964 aws_sdk_lambda::types::ExecutionDetails::builder()
1965 .input_payload("{}")
1966 .build(),
1967 )
1968 .build()
1969 .expect("id, type, status and start timestamp are set");
1970 let execution_info = execution_info_from_operation(&execution);
1971 let first_page = BootstrapState {
1972 log: operations_to_checkpoint_log(&[execution, make_step_op("s1", "1")]),
1973 input_payload: Some("{}".to_owned()),
1974 execution: execution_info,
1975 };
1976
1977 let BootstrapState {
1978 log,
1979 input_payload: input,
1980 ..
1981 } = resolve_bootstrap_log(&client, "arn:test", "tok", first_page, Some("m-1"))
1982 .await
1983 .expect("bootstrap succeeds");
1984
1985 let markers = client
1986 .markers
1987 .lock()
1988 .unwrap_or_else(std::sync::PoisonError::into_inner)
1989 .clone();
1990 assert_eq!(markers, [Some("m-1".to_owned()), Some("m-2".to_owned())]);
1991 for id in ["s1", "s2", "s3"] {
1992 assert!(log.get(id).is_some(), "{id} must be in the merged log");
1993 }
1994 assert!(
1995 log.get("s0").is_none(),
1996 "the first page must not be re-read"
1997 );
1998 assert_eq!(input.as_deref(), Some("{}"));
1999 }
2000
2001 fn make_execution_op(id: &str, input: Option<&str>) -> Operation {
2004 let mut details = aws_sdk_lambda::types::ExecutionDetails::builder();
2005 if let Some(input) = input {
2006 details = details.input_payload(input);
2007 }
2008 Operation::builder()
2009 .id(id)
2010 .r#type(OperationType::Execution)
2011 .status(aws_sdk_lambda::types::OperationStatus::Started)
2012 .start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
2013 .execution_details(details.build())
2014 .build()
2015 .expect("id, type, status and start timestamp are set")
2016 }
2017
2018 struct MarkerPagesClient {
2022 pages: Vec<(&'static str, Vec<Operation>, Option<&'static str>)>,
2023 markers: Mutex<Vec<Option<String>>>,
2024 }
2025
2026 impl ExecutionClient for MarkerPagesClient {
2027 fn checkpoint(
2028 &self,
2029 _execution_arn: &str,
2030 _checkpoint_token: &str,
2031 _updates: Vec<OperationUpdate>,
2032 ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
2033 {
2034 Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
2035 }
2036
2037 fn get_state(
2038 &self,
2039 _execution_arn: &str,
2040 _checkpoint_token: &str,
2041 marker: Option<&str>,
2042 ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
2043 {
2044 self.markers
2045 .lock()
2046 .unwrap_or_else(std::sync::PoisonError::into_inner)
2047 .push(marker.map(str::to_owned));
2048 let page = self
2049 .pages
2050 .iter()
2051 .find(|(key, _, _)| Some(*key) == marker)
2052 .map_or_else(
2053 || GetStateOutput::new(vec![], None),
2054 |(_, ops, next)| GetStateOutput::new(ops.clone(), next.map(str::to_owned)),
2055 );
2056 Box::pin(async move { Ok(page) })
2057 }
2058 }
2059
2060 async fn resolve_input_from_pages(
2064 pages: Vec<(&'static str, Vec<Operation>, Option<&'static str>)>,
2065 ) -> (Option<String>, Vec<Option<String>>) {
2066 let client = MarkerPagesClient {
2067 pages,
2068 markers: Mutex::new(Vec::new()),
2069 };
2070 let BootstrapState {
2071 input_payload: input,
2072 ..
2073 } = resolve_bootstrap_log(
2074 &client,
2075 "arn:test",
2076 "tok",
2077 BootstrapState {
2078 log: operations_to_checkpoint_log(&[]),
2079 input_payload: None,
2080 execution: None,
2081 },
2082 Some("m-1"),
2083 )
2084 .await
2085 .expect("bootstrap succeeds");
2086 let markers = client
2087 .markers
2088 .lock()
2089 .unwrap_or_else(std::sync::PoisonError::into_inner)
2090 .clone();
2091 (input, markers)
2092 }
2093
2094 #[tokio::test]
2101 async fn bootstrap_input_comes_from_first_fetched_execution_operation() {
2102 let (input, markers) = resolve_input_from_pages(vec![
2103 ("m-1", vec![make_step_op("s1", "1")], Some("m-2")),
2104 (
2105 "m-2",
2106 vec![make_execution_op("execution", Some(r#"{"count":1}"#))],
2107 Some("m-3"),
2108 ),
2109 (
2110 "m-3",
2111 vec![make_execution_op("late", Some(r#"{"late":1}"#))],
2112 None,
2113 ),
2114 ])
2115 .await;
2116 assert_eq!(input.as_deref(), Some(r#"{"count":1}"#));
2117 assert_eq!(
2118 markers,
2119 [
2120 Some("m-1".to_owned()),
2121 Some("m-2".to_owned()),
2122 Some("m-3".to_owned())
2123 ]
2124 );
2125
2126 let (input, markers) = resolve_input_from_pages(vec![
2127 ("m-1", vec![make_step_op("s1", "1")], Some("m-2")),
2128 (
2129 "m-2",
2130 vec![make_execution_op("execution", None)],
2131 Some("m-3"),
2132 ),
2133 (
2134 "m-3",
2135 vec![make_execution_op("late", Some(r#"{"late":1}"#))],
2136 None,
2137 ),
2138 ])
2139 .await;
2140 assert_eq!(input, None, "an absent first payload must not be replaced");
2141 assert_eq!(markers.len(), 3, "pagination still reaches the last page");
2142 }
2143
2144 #[test]
2148 fn output_accessors_read_empty_values_as_none() {
2149 let output = CheckpointOutput::new(Some(String::new()), vec![], None);
2150 assert_eq!(output.checkpoint_token(), None);
2151 let page = GetStateOutput::new(vec![], Some(String::new()));
2152 assert_eq!(page.next_marker(), None);
2153 }
2154
2155 struct EmptyMarkerClient {
2157 calls: Mutex<u32>,
2158 }
2159
2160 impl ExecutionClient for EmptyMarkerClient {
2161 fn checkpoint(
2162 &self,
2163 _execution_arn: &str,
2164 _checkpoint_token: &str,
2165 _updates: Vec<OperationUpdate>,
2166 ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
2167 {
2168 Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
2169 }
2170
2171 fn get_state(
2172 &self,
2173 _execution_arn: &str,
2174 _checkpoint_token: &str,
2175 _marker: Option<&str>,
2176 ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
2177 {
2178 *self
2179 .calls
2180 .lock()
2181 .unwrap_or_else(std::sync::PoisonError::into_inner) += 1;
2182 let page = GetStateOutput::new(vec![make_step_op("a", "1")], Some(String::new()));
2183 Box::pin(async move { Ok(page) })
2184 }
2185 }
2186
2187 #[tokio::test]
2190 async fn sdk_stops_paginating_on_empty_marker() {
2191 let client = EmptyMarkerClient {
2192 calls: Mutex::new(0),
2193 };
2194 let operations = get_full_state(&client, "arn:test", "tok")
2195 .await
2196 .expect("whole state");
2197
2198 let ids: Vec<&str> = operations.iter().map(Operation::id).collect();
2199 assert_eq!(ids, ["a"]);
2200 assert_eq!(
2201 *client
2202 .calls
2203 .lock()
2204 .unwrap_or_else(std::sync::PoisonError::into_inner),
2205 1
2206 );
2207 }
2208
2209 #[tokio::test]
2214 async fn checkpoint_maps_empty_token_and_marker_to_none() {
2215 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_output(|| {
2216 CheckpointDurableExecutionOutput::builder()
2217 .checkpoint_token("")
2218 .new_execution_state(
2219 CheckpointUpdatedExecutionState::builder()
2220 .operations(make_step_op("op-1", r#""r1""#))
2221 .next_marker("")
2222 .build(),
2223 )
2224 .build()
2225 });
2226 let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
2227
2228 let output = client
2229 .checkpoint("arn:test", "tok", Vec::new())
2230 .await
2231 .expect("mocked checkpoint succeeds");
2232
2233 assert_eq!(output.checkpoint_token, None);
2234 assert_eq!(output.next_marker, None);
2235 assert_eq!(output.updated_operations.len(), 1);
2236 }
2237
2238 #[tokio::test]
2240 async fn get_state_maps_empty_marker_to_none() {
2241 let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_output(|| {
2242 GetDurableExecutionStateOutput::builder()
2243 .operations(make_step_op("op-1", r#""p1""#))
2244 .next_marker("")
2245 .build()
2246 .expect("operations is set")
2247 });
2248 let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
2249
2250 let page = client
2251 .get_state("arn:test", "tok", None)
2252 .await
2253 .expect("get_state succeeds");
2254
2255 assert_eq!(page.next_marker, None);
2256 assert_eq!(page.operations.len(), 1);
2257 }
2258
2259 #[tokio::test]
2262 async fn get_state_final_error_maps_to_retryable_client_error() {
2263 let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_error(|| {
2264 GetDurableExecutionStateError::InvalidParameterValueException(
2265 aws_sdk_lambda::types::error::InvalidParameterValueException::builder().build(),
2266 )
2267 });
2268 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2269 let client = LambdaExecutionClient::new(sdk_client);
2270
2271 let err = client
2272 .get_state("arn:test", "tok", None)
2273 .await
2274 .expect_err("modeled error must fail the call");
2275 assert!(err.is_retryable());
2276 }
2277
2278 #[tokio::test]
2282 async fn checkpoint_unmodeled_403_fails_the_execution() {
2283 let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
2284 .sequence()
2285 .http_status(403, None)
2286 .repeatedly()
2287 .build();
2288 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2289 let client = LambdaExecutionClient::new(sdk_client);
2290
2291 let err = client
2292 .checkpoint("arn:test", "tok", Vec::new())
2293 .await
2294 .expect_err("a 403 must fail the call");
2295 assert!(!err.is_retryable());
2296 assert_eq!(rule.num_calls(), 1, "a 403 is not retried");
2297 }
2298
2299 #[tokio::test]
2303 async fn get_state_kms_errors_classify_non_retryable() {
2304 use aws_sdk_lambda::types::error::{
2305 KmsAccessDeniedException, KmsDisabledException, KmsInvalidStateException,
2306 KmsNotFoundException,
2307 };
2308 type Make = fn() -> GetDurableExecutionStateError;
2309 let cases: [(Make, &str); 4] = [
2310 (
2311 || {
2312 GetDurableExecutionStateError::KmsAccessDeniedException(
2313 KmsAccessDeniedException::builder().build(),
2314 )
2315 },
2316 "KMSAccessDeniedException",
2317 ),
2318 (
2319 || {
2320 GetDurableExecutionStateError::KmsDisabledException(
2321 KmsDisabledException::builder().build(),
2322 )
2323 },
2324 "KMSDisabledException",
2325 ),
2326 (
2327 || {
2328 GetDurableExecutionStateError::KmsInvalidStateException(
2329 KmsInvalidStateException::builder().build(),
2330 )
2331 },
2332 "KMSInvalidStateException",
2333 ),
2334 (
2335 || {
2336 GetDurableExecutionStateError::KmsNotFoundException(
2337 KmsNotFoundException::builder().build(),
2338 )
2339 },
2340 "KMSNotFoundException",
2341 ),
2342 ];
2343 for (make, wire_name) in cases {
2344 let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_error(make);
2345 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2346 let client = LambdaExecutionClient::new(sdk_client);
2347
2348 let err = client
2349 .get_state("arn:test", "tok", None)
2350 .await
2351 .expect_err("a KMS error must fail the call");
2352 assert!(!err.is_retryable(), "{wire_name} must be non-retryable");
2353 assert!(
2354 err.to_string().contains(wire_name),
2355 "message must name {wire_name}, got: {err}"
2356 );
2357 }
2358 }
2359
2360 #[tokio::test]
2363 async fn get_state_plain_4xx_stays_retryable() {
2364 let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
2365 .sequence()
2366 .http_status(403, None)
2367 .repeatedly()
2368 .build();
2369 let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2370 let client = LambdaExecutionClient::new(sdk_client);
2371
2372 let err = client
2373 .get_state("arn:test", "tok", None)
2374 .await
2375 .expect_err("a 403 must fail the call");
2376 assert!(err.is_retryable());
2377 }
2378
2379 #[tokio::test]
2384 async fn accept_then_retryable_error_records_then_fails() {
2385 let client = InMemoryExecutionClient::new(Vec::new());
2386 client.enqueue_checkpoint_response(TestResponse::AcceptThenRetryableError(
2387 "lost response".to_owned(),
2388 ));
2389 let update = OperationUpdate::builder()
2390 .id("e")
2391 .r#type(OperationType::Execution)
2392 .action(OperationAction::Succeed)
2393 .build()
2394 .expect("valid update");
2395 let err = client
2396 .checkpoint("arn", "token", vec![update])
2397 .await
2398 .expect_err("a retryable error");
2399 assert!(err.is_retryable());
2400 assert_eq!(err.message(), "lost response");
2401 assert_eq!(client.recorded_updates().len(), 1);
2402 }
2403
2404 #[tokio::test]
2405 async fn in_memory_client_returns_queued_responses_in_order() {
2406 let client = InMemoryExecutionClient::new(Vec::new());
2407 client.enqueue_checkpoint_response(TestResponse::RetryableError("throttled".to_owned()));
2408 client.enqueue_checkpoint_response(TestResponse::NonRetryableError("invalid".to_owned()));
2409 client.enqueue_checkpoint_response(TestResponse::Success(Vec::new()));
2410
2411 let first = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2412 let first_err = first.unwrap_err();
2413 assert!(first_err.is_retryable());
2414
2415 let second = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2416 let second_err = second.unwrap_err();
2417 assert!(!second_err.is_retryable());
2418
2419 let third = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2420 assert!(third.is_ok());
2421
2422 let count = *client
2423 .checkpoint_call_count
2424 .lock()
2425 .unwrap_or_else(std::sync::PoisonError::into_inner);
2426 assert_eq!(count, 3);
2427 }
2428
2429 #[tokio::test]
2430 async fn success_returns_checkpoint_output() {
2431 let client = InMemoryExecutionClient::new(Vec::new());
2432 let result = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2433
2434 assert!(result.is_ok());
2435 let output = result.unwrap();
2436 assert!(
2437 output
2438 .checkpoint_token()
2439 .is_some_and(|t| t.starts_with("token-"))
2440 );
2441 }
2442
2443 #[test]
2446 fn checkpoint_log_from_operations_maps_statuses() {
2447 let ops = vec![
2448 Operation::builder()
2449 .id("op-1")
2450 .r#type(OperationType::Step)
2451 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2452 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2453 .step_details(
2454 aws_sdk_lambda::types::StepDetails::builder()
2455 .attempt(1)
2456 .result(r#""hello""#)
2457 .build(),
2458 )
2459 .build()
2460 .unwrap(),
2461 Operation::builder()
2462 .id("op-2")
2463 .r#type(OperationType::Step)
2464 .status(aws_sdk_lambda::types::OperationStatus::Failed)
2465 .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2466 .step_details(
2467 aws_sdk_lambda::types::StepDetails::builder()
2468 .attempt(2)
2469 .error(
2470 aws_sdk_lambda::types::ErrorObject::builder()
2471 .error_type("StepError")
2472 .error_message("oops")
2473 .build(),
2474 )
2475 .build(),
2476 )
2477 .build()
2478 .unwrap(),
2479 ];
2480
2481 let log = operations_to_checkpoint_log(&ops);
2482
2483 let r1 = log.get("op-1");
2484 assert!(r1.is_some());
2485 let r1 = r1.unwrap();
2486 assert_eq!(r1.status, CheckpointStatus::Succeeded);
2487 assert_eq!(r1.result.as_deref(), Some(r#""hello""#));
2488 assert_eq!(r1.attempt, 1);
2489
2490 let r2 = log.get("op-2");
2491 assert!(r2.is_some());
2492 let r2 = r2.unwrap();
2493 assert_eq!(r2.status, CheckpointStatus::Failed);
2494 assert_eq!(r2.error_type.as_deref(), Some("StepError"));
2495 assert_eq!(r2.error_message.as_deref(), Some("oops"));
2496 assert_eq!(r2.attempt, 2);
2497 }
2498
2499 #[test]
2505 fn chained_invoke_maps_to_invoke_specific_fields() {
2506 let ops = vec![
2507 Operation::builder()
2508 .id("invoke-ok")
2509 .r#type(OperationType::ChainedInvoke)
2510 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2511 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2512 .chained_invoke_details(
2513 aws_sdk_lambda::types::ChainedInvokeDetails::builder()
2514 .result(r#""invoke-payload""#)
2515 .build(),
2516 )
2517 .build()
2518 .unwrap(),
2519 Operation::builder()
2520 .id("invoke-err")
2521 .r#type(OperationType::ChainedInvoke)
2522 .status(aws_sdk_lambda::types::OperationStatus::Failed)
2523 .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2524 .chained_invoke_details(
2525 aws_sdk_lambda::types::ChainedInvokeDetails::builder()
2526 .error(
2527 aws_sdk_lambda::types::ErrorObject::builder()
2528 .error_type("InvokeError")
2529 .error_message("downstream failed")
2530 .build(),
2531 )
2532 .build(),
2533 )
2534 .build()
2535 .unwrap(),
2536 ];
2537
2538 let log = operations_to_checkpoint_log(&ops);
2539
2540 let ok = log.get("invoke-ok");
2541 assert!(ok.is_some());
2542 let ok = ok.unwrap();
2543 assert_eq!(ok.invoke_result.as_deref(), Some(r#""invoke-payload""#));
2544 assert_eq!(
2545 ok.result, None,
2546 "invoke results must not leak into the generic result field"
2547 );
2548
2549 let err = log.get("invoke-err");
2550 assert!(err.is_some());
2551 let err = err.unwrap();
2552 assert_eq!(err.invoke_error_type.as_deref(), Some("InvokeError"));
2553 assert_eq!(
2554 err.invoke_error_message.as_deref(),
2555 Some("downstream failed")
2556 );
2557 assert_eq!(
2558 err.error_type, None,
2559 "invoke errors must not leak into the generic error fields"
2560 );
2561 assert_eq!(err.error_message, None);
2562 }
2563
2564 #[test]
2568 fn context_replay_children_is_preserved() {
2569 let ops = vec![
2570 Operation::builder()
2571 .id("child-replay")
2572 .r#type(OperationType::Context)
2573 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2574 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2575 .context_details(
2576 aws_sdk_lambda::types::ContextDetails::builder()
2577 .replay_children(true)
2578 .build(),
2579 )
2580 .build()
2581 .unwrap(),
2582 Operation::builder()
2583 .id("child-inline")
2584 .r#type(OperationType::Context)
2585 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2586 .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2587 .context_details(
2588 aws_sdk_lambda::types::ContextDetails::builder()
2589 .result(r#""child-result""#)
2590 .build(),
2591 )
2592 .build()
2593 .unwrap(),
2594 ];
2595
2596 let log = operations_to_checkpoint_log(&ops);
2597
2598 let replayed = log.get("child-replay");
2599 assert!(replayed.is_some());
2600 let replayed = replayed.unwrap();
2601 assert!(
2602 replayed.replay_children,
2603 "ReplayChildren must survive the typed-state conversion"
2604 );
2605
2606 let inline = log.get("child-inline");
2607 assert!(inline.is_some());
2608 let inline = inline.unwrap();
2609 assert!(!inline.replay_children);
2610 assert_eq!(inline.result.as_deref(), Some(r#""child-result""#));
2611 }
2612
2613 #[test]
2616 fn execution_operations_are_filtered_from_log() {
2617 let ops = vec![
2618 Operation::builder()
2619 .id("exec-0")
2620 .r#type(OperationType::Execution)
2621 .status(aws_sdk_lambda::types::OperationStatus::Started)
2622 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2623 .build()
2624 .unwrap(),
2625 Operation::builder()
2626 .id("step-1")
2627 .r#type(OperationType::Step)
2628 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2629 .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2630 .build()
2631 .unwrap(),
2632 ];
2633
2634 let log = operations_to_checkpoint_log(&ops);
2635
2636 assert!(
2637 log.get("exec-0").is_none(),
2638 "Execution operations must be filtered like the JSON path does"
2639 );
2640 assert!(log.get("step-1").is_some());
2641 }
2642
2643 #[tokio::test]
2646 async fn get_state_returns_preloaded_operations() {
2647 let ops = vec![
2648 Operation::builder()
2649 .id("op-1")
2650 .r#type(OperationType::Step)
2651 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2652 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2653 .build()
2654 .unwrap(),
2655 ];
2656
2657 let client = InMemoryExecutionClient::new(ops);
2658 let result = client.get_state("arn:test", "token-0", None).await;
2659
2660 assert!(result.is_ok());
2661 let output = result.unwrap();
2662 assert_eq!(output.operations.len(), 1);
2663 assert_eq!(output.operations.first().map(Operation::id), Some("op-1"));
2664 }
2665
2666 #[test]
2669 fn merge_operations_inserts_callback_id_into_existing_log() {
2670 let log = CheckpointLog::from_records(vec![(
2672 "op-cb".to_owned(),
2673 CheckpointRecord {
2674 id: "op-cb".to_owned(),
2675 status: CheckpointStatus::Started,
2676 result: None,
2677 error_type: None,
2678 error_message: None,
2679 error_data: None,
2680 stack_trace: None,
2681 attempt: 0,
2682 invoke_result: None,
2683 invoke_error_type: None,
2684 invoke_error_message: None,
2685 invoke_error_data: None,
2686 invoke_stack_trace: None,
2687 replay_children: false,
2688 callback_id: None,
2689 op_type: None,
2690 sub_type: None,
2691 due_at: None,
2692 op_name: None,
2693 parent_id: None,
2694 start_timestamp: None,
2695 end_timestamp: None,
2696 },
2697 )]);
2698
2699 let updated_ops = vec![
2701 Operation::builder()
2702 .id("op-cb")
2703 .r#type(OperationType::Callback)
2704 .status(aws_sdk_lambda::types::OperationStatus::Pending)
2705 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2706 .callback_details(
2707 aws_sdk_lambda::types::CallbackDetails::builder()
2708 .callback_id("assigned-cb-id-42")
2709 .build(),
2710 )
2711 .build()
2712 .unwrap(),
2713 ];
2714
2715 merge_operations_into_log(&log, &updated_ops);
2716
2717 let record = log.get("op-cb");
2718 assert!(record.is_some());
2719 let record = record.unwrap();
2720 assert_eq!(record.callback_id.as_deref(), Some("assigned-cb-id-42"));
2721 assert_eq!(record.status, CheckpointStatus::Pending);
2722 }
2723
2724 #[test]
2725 fn merge_operations_adds_new_record_to_log() {
2726 let log = CheckpointLog::empty();
2727
2728 let updated_ops = vec![
2730 Operation::builder()
2731 .id("new-op")
2732 .r#type(OperationType::Step)
2733 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2734 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2735 .step_details(
2736 aws_sdk_lambda::types::StepDetails::builder()
2737 .result(r#""merged result""#)
2738 .build(),
2739 )
2740 .build()
2741 .unwrap(),
2742 ];
2743
2744 merge_operations_into_log(&log, &updated_ops);
2745
2746 let record = log.get("new-op");
2747 assert!(record.is_some());
2748 let record = record.unwrap();
2749 assert_eq!(record.status, CheckpointStatus::Succeeded);
2750 assert_eq!(record.result.as_deref(), Some(r#""merged result""#));
2751 }
2752
2753 fn status_op(
2756 id: &str,
2757 ty: OperationType,
2758 status: aws_sdk_lambda::types::OperationStatus,
2759 callback_id: Option<&str>,
2760 ) -> Operation {
2761 let mut builder = Operation::builder()
2762 .id(id)
2763 .r#type(ty)
2764 .status(status)
2765 .start_timestamp(aws_smithy_types::DateTime::from_secs(0));
2766 if let Some(callback_id) = callback_id {
2767 builder = builder.callback_details(
2768 aws_sdk_lambda::types::CallbackDetails::builder()
2769 .callback_id(callback_id)
2770 .build(),
2771 );
2772 }
2773 builder.build().unwrap()
2774 }
2775
2776 #[test]
2780 fn merge_reports_status_changes() {
2781 use aws_sdk_lambda::types::OperationStatus as S;
2782 let log = CheckpointLog::from_records(vec![
2783 (
2784 "b".to_owned(),
2785 CheckpointRecord::with_status("b", CheckpointStatus::Started),
2786 ),
2787 (
2788 "c".to_owned(),
2789 CheckpointRecord::with_status("c", CheckpointStatus::Started),
2790 ),
2791 ]);
2792 let changed = merge_operations_into_log(
2793 &log,
2794 &[
2795 status_op("a", OperationType::Step, S::Started, None),
2796 status_op("b", OperationType::Callback, S::Started, Some("cb-1")),
2797 status_op("c", OperationType::Step, S::Succeeded, None),
2798 status_op("exec", OperationType::Execution, S::Started, None),
2799 ],
2800 );
2801 assert_eq!(changed, ["a", "c"]);
2802 assert_eq!(
2803 log.get("b").unwrap().callback_id.as_deref(),
2804 Some("cb-1"),
2805 "the field-only change is still merged"
2806 );
2807 assert!(!log.contains("exec"));
2808 }
2809
2810 #[test]
2813 fn merge_compares_unknown_statuses_by_token() {
2814 use aws_sdk_lambda::types::OperationStatus as S;
2815 let log = CheckpointLog::from_records(vec![(
2816 "u".to_owned(),
2817 CheckpointRecord::with_status("u", CheckpointStatus::Unknown("A".to_owned())),
2818 )]);
2819 let op = |token: &str| status_op("u", OperationType::Step, S::from(token), None);
2820 assert_eq!(
2821 merge_operations_into_log(&log, &[op("A")]),
2822 Vec::<String>::new()
2823 );
2824 assert_eq!(merge_operations_into_log(&log, &[op("B")]), ["u"]);
2825 }
2826
2827 fn make_step_op(id: &str, result: &str) -> Operation {
2831 Operation::builder()
2832 .id(id)
2833 .r#type(OperationType::Step)
2834 .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2835 .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2836 .step_details(
2837 aws_sdk_lambda::types::StepDetails::builder()
2838 .result(result)
2839 .build(),
2840 )
2841 .build()
2842 .unwrap()
2843 }
2844
2845 #[tokio::test]
2848 async fn in_memory_get_state_returns_all_operations() {
2849 let ops = vec![
2850 make_step_op("step-1", "\"page1\""),
2851 make_step_op("step-2", "\"page2\""),
2852 make_step_op("step-3", "\"page3\""),
2853 ];
2854 let client = InMemoryExecutionClient::new(ops);
2855
2856 let state = client.get_state("arn:test", "token-0", None).await;
2857 assert!(state.is_ok());
2858 let state = state.unwrap();
2859 assert_eq!(state.operations.len(), 3);
2860 }
2861
2862 #[tokio::test]
2866 async fn checkpoint_with_marker_returns_paginated_response() {
2867 let all_ops = vec![
2868 make_step_op("step-1", "\"r1\""),
2869 make_step_op("step-2", "\"r2\""),
2870 make_step_op("step-3", "\"r3\""),
2871 ];
2872 let client = InMemoryExecutionClient::new(all_ops);
2873
2874 let page1_ops = vec![make_step_op("step-1", "\"r1\"")];
2876 client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
2877 page1_ops,
2878 "marker-page-2".to_owned(),
2879 ));
2880
2881 let result = client.checkpoint("arn:test", "token-0", vec![]).await;
2882 assert!(result.is_ok());
2883 let output = result.unwrap();
2884 assert_eq!(output.updated_operations.len(), 1);
2885 assert_eq!(output.next_marker, Some("marker-page-2".to_owned()));
2886
2887 let full_state = client
2889 .get_state(
2890 "arn:test",
2891 output.checkpoint_token().unwrap_or_default(),
2892 None,
2893 )
2894 .await;
2895 assert!(full_state.is_ok());
2896 let full_state = full_state.unwrap();
2897 assert_eq!(full_state.operations.len(), 3);
2898 }
2899
2900 #[test]
2903 fn operations_to_checkpoint_log_handles_multi_page_operations() {
2904 let ops = vec![
2905 make_step_op("step-1", "\"result-1\""),
2906 make_step_op("step-2", "\"result-2\""),
2907 make_step_op("step-3", "\"result-3\""),
2908 ];
2909
2910 let log = operations_to_checkpoint_log(&ops);
2911
2912 assert!(log.get("step-1").is_some());
2914 assert!(log.get("step-2").is_some());
2915 assert!(log.get("step-3").is_some());
2916
2917 let r1 = log.get("step-1").unwrap();
2918 assert_eq!(r1.result.as_deref(), Some("\"result-1\""));
2919 assert_eq!(r1.status, CheckpointStatus::Succeeded);
2920
2921 let r3 = log.get("step-3").unwrap();
2922 assert_eq!(r3.result.as_deref(), Some("\"result-3\""));
2923 }
2924
2925 use aws_sdk_lambda::config::http::HttpResponse;
2928 use aws_sdk_lambda::error::SdkError;
2929 use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError as CkptErr;
2930 use aws_smithy_runtime_api::http::StatusCode;
2931 use aws_smithy_types::body::SdkBody;
2932
2933 fn raw_response(status: u16) -> HttpResponse {
2935 HttpResponse::new(
2936 StatusCode::try_from(status).expect("valid HTTP status"),
2937 SdkBody::empty(),
2938 )
2939 }
2940
2941 fn classify<E: Into<CkptErr>>(err: E, status: u16) -> ExecutionClientError {
2943 classify_checkpoint_error(SdkError::service_error(err.into(), raw_response(status)))
2944 }
2945
2946 fn invalid_parameter(message: Option<&str>) -> CkptErr {
2947 let mut builder = aws_sdk_lambda::types::error::InvalidParameterValueException::builder();
2948 if let Some(m) = message {
2949 builder = builder.message(m);
2950 }
2951 CkptErr::InvalidParameterValueException(builder.build())
2952 }
2953
2954 #[test]
2955 fn throttling_and_5xx_classify_retryable() {
2956 let throttled = classify(
2957 CkptErr::TooManyRequestsException(
2958 aws_sdk_lambda::types::error::TooManyRequestsException::builder().build(),
2959 ),
2960 429,
2961 );
2962 assert!(throttled.is_retryable());
2963
2964 let server = classify(
2965 CkptErr::ServiceException(
2966 aws_sdk_lambda::types::error::ServiceException::builder().build(),
2967 ),
2968 500,
2969 );
2970 assert!(server.is_retryable());
2971 }
2972
2973 #[test]
2974 fn invalid_parameter_classifies_non_retryable() {
2975 assert!(!classify(invalid_parameter(Some("payload too large")), 400).is_retryable());
2976 assert!(!classify(invalid_parameter(None), 400).is_retryable());
2978 }
2979
2980 #[test]
2987 fn stale_token_rejection_classifies_retryable() {
2988 let stale = classify(
2989 invalid_parameter(Some("Invalid checkpoint token: token has been superseded")),
2990 400,
2991 );
2992 assert!(stale.is_retryable());
2993
2994 let not_prefix = classify(
2995 invalid_parameter(Some(
2996 "field X rejected (not an Invalid checkpoint token case)",
2997 )),
2998 400,
2999 );
3000 assert!(!not_prefix.is_retryable());
3001
3002 let wrong_case = classify(
3003 invalid_parameter(Some("Invalid Checkpoint Token: old non-service spelling")),
3004 400,
3005 );
3006 assert!(!wrong_case.is_retryable());
3007 }
3008
3009 #[test]
3010 fn kms_misconfiguration_classifies_non_retryable() {
3011 let kms = classify(
3012 CkptErr::KmsAccessDeniedException(
3013 aws_sdk_lambda::types::error::KmsAccessDeniedException::builder().build(),
3014 ),
3015 502,
3016 );
3017 assert!(!kms.is_retryable());
3018 }
3019
3020 #[test]
3024 fn unhandled_4xx_classifies_non_retryable() {
3025 for status in [400, 403, 404, 409, 413] {
3026 let err = classify(CkptErr::unhandled("unmodeled rejection"), status);
3027 assert!(!err.is_retryable(), "status {status} must be non-retryable");
3028 }
3029 }
3030
3031 #[test]
3033 fn unhandled_429_classifies_retryable() {
3034 assert!(classify(CkptErr::unhandled("throttled"), 429).is_retryable());
3035 }
3036
3037 #[test]
3043 fn unhandled_5xx_classifies_retryable() {
3044 for status in [500, 502, 503] {
3045 let err = classify(CkptErr::unhandled("some new service error"), status);
3046 assert!(err.is_retryable(), "status {status} must be retryable");
3047 }
3048 }
3049
3050 #[test]
3053 fn response_error_4xx_classifies_non_retryable() {
3054 let err: SdkError<CkptErr, HttpResponse> =
3055 SdkError::response_error("unparseable body", raw_response(403));
3056 assert!(!classify_checkpoint_error(err).is_retryable());
3057 }
3058
3059 #[test]
3061 fn response_error_200_classifies_retryable() {
3062 let err: SdkError<CkptErr, HttpResponse> =
3063 SdkError::response_error("unparseable body", raw_response(200));
3064 assert!(classify_checkpoint_error(err).is_retryable());
3065 }
3066
3067 #[test]
3068 fn transport_failures_classify_retryable() {
3069 let timeout: SdkError<CkptErr, HttpResponse> = SdkError::timeout_error("connect timed out");
3070 assert!(classify_checkpoint_error(timeout).is_retryable());
3071 }
3072
3073 #[test]
3087 fn checkpoint_error_variant_canary() {
3088 const CLASSIFIED_VARIANTS: &[&str] = &[
3089 "InvalidParameterValueException",
3090 "KmsAccessDeniedException",
3091 "KmsDisabledException",
3092 "KmsInvalidStateException",
3093 "KmsNotFoundException",
3094 "ServiceException",
3095 "TooManyRequestsException",
3096 "Unhandled",
3097 ];
3098
3099 let source = read_sdk_source("src/operation/checkpoint_durable_execution.rs");
3100 let mut actual = extract_enum_variants(&source, "CheckpointDurableExecutionError");
3101 actual.sort_unstable();
3102
3103 let mut expected: Vec<String> = CLASSIFIED_VARIANTS
3104 .iter()
3105 .map(|s| (*s).to_owned())
3106 .collect();
3107 expected.sort_unstable();
3108
3109 assert_eq!(
3110 actual, expected,
3111 "aws-sdk-lambda's CheckpointDurableExecutionError variant set changed. \
3112 Reclassify the new/removed variants in classify_checkpoint_error \
3113 (src/client.rs), then update this canary's pinned list."
3114 );
3115 }
3116
3117 #[test]
3130 #[expect(clippy::too_many_lines)] fn upstream_type_enum_variant_canary() {
3132 struct Pin {
3133 file: &'static str,
3134 enum_name: &'static str,
3135 variants: &'static [&'static str],
3136 wildcard_sites: &'static str,
3137 }
3138 let pins = [
3139 Pin {
3140 file: "src/types/_operation_status.rs",
3141 enum_name: "OperationStatus",
3142 variants: &[
3143 "Cancelled",
3144 "Failed",
3145 "Pending",
3146 "Ready",
3147 "Started",
3148 "Stopped",
3149 "Succeeded",
3150 "TimedOut",
3151 "Unknown",
3152 ],
3153 wildcard_sites: "sdk_status_to_checkpoint_status (src/client.rs); \
3154 operation_status_str and operation_status_wire_str \
3155 (src/test_util.rs)",
3156 },
3157 Pin {
3158 file: "src/types/_operation_type.rs",
3159 enum_name: "OperationType",
3160 variants: &[
3161 "Callback",
3162 "ChainedInvoke",
3163 "Context",
3164 "Execution",
3165 "Step",
3166 "Unknown",
3167 "Wait",
3168 ],
3169 wildcard_sites: "operation_type_str (src/tracing_layer.rs); \
3170 operation_type_str (src/test_util.rs)",
3171 },
3172 Pin {
3173 file: "src/types/_operation_action.rs",
3174 enum_name: "OperationAction",
3175 variants: &["Cancel", "Fail", "Retry", "Start", "Succeed", "Unknown"],
3176 wildcard_sites: "PendingTransitionEvent::emit (src/tracing_layer.rs); \
3177 BackendState::apply_update (src/test_util.rs); \
3178 child_starts_before_first_terminal \
3179 (src/map_parallel.rs tests)",
3180 },
3181 Pin {
3182 file: "src/types/_execution_status.rs",
3183 enum_name: "ExecutionStatus",
3184 variants: &[
3185 "Failed",
3186 "Running",
3187 "Stopped",
3188 "Succeeded",
3189 "TimedOut",
3190 "Unknown",
3191 ],
3192 wildcard_sites: "execution_status_str (src/test_util.rs)",
3193 },
3194 Pin {
3195 file: "src/types/_event_type.rs",
3196 enum_name: "EventType",
3197 variants: &[
3198 "CallbackFailed",
3199 "CallbackStarted",
3200 "CallbackSucceeded",
3201 "CallbackTimedOut",
3202 "ChainedInvokeFailed",
3203 "ChainedInvokeStarted",
3204 "ChainedInvokeStopped",
3205 "ChainedInvokeSucceeded",
3206 "ChainedInvokeTimedOut",
3207 "ContextFailed",
3208 "ContextStarted",
3209 "ContextSucceeded",
3210 "ExecutionFailed",
3211 "ExecutionStarted",
3212 "ExecutionStopped",
3213 "ExecutionSucceeded",
3214 "ExecutionTimedOut",
3215 "InvocationCompleted",
3216 "StepFailed",
3217 "StepStarted",
3218 "StepSucceeded",
3219 "Unknown",
3220 "WaitCancelled",
3221 "WaitStarted",
3222 "WaitSucceeded",
3223 ],
3224 wildcard_sites: "apply_event_to_op and event_op_type (src/test_util.rs)",
3225 },
3226 ];
3227
3228 for pin in pins {
3229 let source = read_sdk_source(pin.file);
3230 let mut actual = extract_enum_variants(&source, pin.enum_name);
3231 actual.sort_unstable();
3232
3233 let mut expected: Vec<String> = pin.variants.iter().map(|s| (*s).to_owned()).collect();
3234 expected.sort_unstable();
3235
3236 assert_eq!(
3237 actual, expected,
3238 "aws-sdk-lambda's {} variant set changed. Classify the new/removed \
3239 variants explicitly at every wildcard match site — {} — then update \
3240 this canary's pinned list.",
3241 pin.enum_name, pin.wildcard_sites
3242 );
3243 }
3244 }
3245
3246 #[test]
3252 fn unknown_operation_status_maps_to_unknown_carrying_raw() {
3253 use aws_sdk_lambda::types::OperationStatus;
3254
3255 let novel = OperationStatus::from("PAUSED");
3256 assert_eq!(
3257 sdk_status_to_checkpoint_status(&novel),
3258 CheckpointStatus::Unknown("PAUSED".to_owned())
3259 );
3260
3261 let modeled = [
3262 (OperationStatus::Started, CheckpointStatus::Started),
3263 (OperationStatus::Pending, CheckpointStatus::Pending),
3264 (OperationStatus::Ready, CheckpointStatus::Ready),
3265 (OperationStatus::Succeeded, CheckpointStatus::Succeeded),
3266 (OperationStatus::Failed, CheckpointStatus::Failed),
3267 (OperationStatus::Cancelled, CheckpointStatus::Cancelled),
3268 (OperationStatus::TimedOut, CheckpointStatus::TimedOut),
3269 (OperationStatus::Stopped, CheckpointStatus::Stopped),
3270 ];
3271 for (sdk, expected) in modeled {
3272 assert_eq!(
3273 sdk_status_to_checkpoint_status(&sdk),
3274 expected,
3275 "modeled variant {sdk:?} must map explicitly"
3276 );
3277 }
3278 }
3279
3280 fn read_sdk_source(relative: &str) -> String {
3284 let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
3285 let lockfile = std::fs::read_to_string(manifest_dir.join("Cargo.lock"))
3286 .expect("Cargo.lock readable at the workspace root");
3287 let version = lockfile
3288 .split("[[package]]")
3289 .find(|block| block.contains("name = \"aws-sdk-lambda\""))
3290 .and_then(|block| {
3291 block
3292 .lines()
3293 .find_map(|line| line.trim().strip_prefix("version = \""))
3294 .map(|rest| rest.trim_end_matches('"').to_owned())
3295 })
3296 .expect("aws-sdk-lambda pinned in Cargo.lock");
3297
3298 let cargo_home = std::env::var_os("CARGO_HOME").map_or_else(
3299 || {
3300 let home = std::env::var_os("HOME").expect("HOME set");
3301 std::path::PathBuf::from(home).join(".cargo")
3302 },
3303 std::path::PathBuf::from,
3304 );
3305 let registry_src = cargo_home.join("registry").join("src");
3306 let crate_dir_name = format!("aws-sdk-lambda-{version}");
3307 let crate_dir = std::fs::read_dir(®istry_src)
3308 .expect("cargo registry src directory readable")
3309 .filter_map(Result::ok)
3310 .map(|entry| entry.path().join(&crate_dir_name))
3311 .find(|candidate| candidate.is_dir())
3312 .expect("pinned aws-sdk-lambda version extracted in the cargo registry");
3313
3314 std::fs::read_to_string(crate_dir.join(relative)).expect("SDK source file readable")
3315 }
3316
3317 #[expect(clippy::panic, clippy::map_unwrap_or)]
3322 fn extract_enum_variants(source: &str, enum_name: &str) -> Vec<String> {
3324 let header = format!("pub enum {enum_name} {{");
3325 let start = source
3326 .find(&header)
3327 .unwrap_or_else(|| panic!("enum {enum_name} present in SDK source"));
3328 let body_start = start + header.len();
3329 let body_end = source[body_start..]
3330 .find("\n}")
3331 .map(|offset| body_start + offset)
3332 .unwrap_or_else(|| panic!("enum {enum_name} body terminated"));
3333 let body = &source[body_start..body_end];
3334
3335 body.lines()
3336 .filter_map(|line| {
3337 let trimmed = line.trim();
3338 let first = trimmed.chars().next()?;
3343 if !first.is_ascii_uppercase() {
3344 return None;
3345 }
3346 let ident: String = trimmed
3347 .chars()
3348 .take_while(char::is_ascii_alphanumeric)
3349 .collect();
3350 let rest = trimmed.get(ident.len()..)?;
3351 if ident.is_empty() || !(rest.starts_with('(') || rest.starts_with(',')) {
3352 return None;
3353 }
3354 Some(ident)
3355 })
3356 .collect()
3357 }
3358
3359 use aws_sdk_lambda::config::{AppName, BehaviorVersion, Credentials, Region};
3367 use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
3368 use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
3369 use aws_smithy_types::retry::RetryConfig;
3370
3371 fn expected_token() -> String {
3372 format!(
3373 "aws-durable-execution-sdk-rust/{}",
3374 env!("CARGO_PKG_VERSION")
3375 )
3376 }
3377
3378 fn replay_event(status: u16, body: &'static str) -> ReplayEvent {
3379 let response = HttpResponse::new(
3380 StatusCode::try_from(status).expect("valid status"),
3381 SdkBody::from(body),
3382 );
3383 ReplayEvent::new(HttpRequest::new(SdkBody::empty()), response)
3384 }
3385
3386 fn replay_lambda_client(
3389 http: &StaticReplayClient,
3390 app_name: Option<AppName>,
3391 ) -> LambdaExecutionClient {
3392 let mut config = aws_sdk_lambda::config::Builder::new()
3393 .behavior_version(BehaviorVersion::latest())
3394 .region(Region::new("us-east-1"))
3395 .credentials_provider(Credentials::new("akid", "secret", None, None, "test"))
3396 .retry_config(
3397 RetryConfig::standard().with_initial_backoff(std::time::Duration::from_millis(1)),
3398 )
3399 .http_client(http.clone());
3400 config.set_app_name(app_name);
3401 LambdaExecutionClient::new(aws_sdk_lambda::Client::from_conf(config.build()))
3402 }
3403
3404 fn user_agent_headers(http: &StaticReplayClient) -> Vec<Vec<String>> {
3407 http.actual_requests()
3408 .map(|req| {
3409 req.headers()
3410 .get_all("user-agent")
3411 .map(str::to_owned)
3412 .collect()
3413 })
3414 .collect()
3415 }
3416
3417 fn assert_one_token(headers: &[String]) {
3420 assert_eq!(
3421 headers.len(),
3422 1,
3423 "exactly one User-Agent header: {headers:?}"
3424 );
3425 let value = headers.first().expect("one header");
3426 let token = expected_token();
3427 let count = value.split(' ').filter(|part| *part == token).count();
3428 assert_eq!(count, 1, "exactly one {token} part in {value:?}");
3429 }
3430
3431 #[tokio::test]
3432 async fn user_agent_token_on_checkpoint() {
3433 let http = StaticReplayClient::new(vec![replay_event(200, r#"{"CheckpointToken":"t-1"}"#)]);
3434 let client = replay_lambda_client(&http, None);
3435
3436 let output = client
3437 .checkpoint("arn:test", "tok", Vec::new())
3438 .await
3439 .expect("replayed checkpoint succeeds");
3440 assert_eq!(output.checkpoint_token(), Some("t-1"));
3441
3442 let attempts = user_agent_headers(&http);
3443 assert_eq!(attempts.len(), 1);
3444 for headers in &attempts {
3445 assert_one_token(headers);
3446 }
3447 }
3448
3449 #[tokio::test]
3450 async fn user_agent_token_on_get_state() {
3451 let http = StaticReplayClient::new(vec![replay_event(200, r#"{"Operations":[]}"#)]);
3452 let client = replay_lambda_client(&http, None);
3453
3454 let output = client
3455 .get_state("arn:test", "tok", None)
3456 .await
3457 .expect("replayed state read succeeds");
3458 assert_eq!(output.next_marker(), None);
3459
3460 let attempts = user_agent_headers(&http);
3461 assert_eq!(attempts.len(), 1, "one get_state call sends one request");
3462 for headers in &attempts {
3463 assert_one_token(headers);
3464 }
3465 }
3466
3467 #[tokio::test]
3468 async fn user_agent_token_once_per_retry_attempt() {
3469 let http = StaticReplayClient::new(vec![
3470 replay_event(503, "{}"),
3471 replay_event(200, r#"{"CheckpointToken":"t-1"}"#),
3472 ]);
3473 let client = replay_lambda_client(&http, None);
3474
3475 client
3476 .checkpoint("arn:test", "tok", Vec::new())
3477 .await
3478 .expect("the retry succeeds");
3479
3480 let attempts = user_agent_headers(&http);
3481 assert_eq!(attempts.len(), 2, "the 503 is retried once");
3482 for headers in &attempts {
3483 assert_one_token(headers);
3484 }
3485 }
3486
3487 #[tokio::test]
3488 async fn user_agent_keeps_app_name() {
3489 let http = StaticReplayClient::new(vec![replay_event(200, r#"{"CheckpointToken":"t-1"}"#)]);
3490 let app_name = AppName::new("my-app").expect("valid app name");
3491 let client = replay_lambda_client(&http, Some(app_name));
3492
3493 client
3494 .checkpoint("arn:test", "tok", Vec::new())
3495 .await
3496 .expect("replayed checkpoint succeeds");
3497
3498 let token = expected_token();
3499 let requests: Vec<_> = http.actual_requests().collect();
3500 assert_eq!(requests.len(), 1);
3501 for req in requests {
3502 let user_agent = req.headers().get("user-agent").expect("User-Agent header");
3503 assert!(
3504 user_agent.split(' ').any(|part| part == "app/my-app"),
3505 "the app name stays in {user_agent:?}"
3506 );
3507 let amz_user_agent = req
3508 .headers()
3509 .get("x-amz-user-agent")
3510 .expect("x-amz-user-agent header");
3511 assert!(
3512 !amz_user_agent.contains(&token),
3513 "x-amz-user-agent stays unchanged: {amz_user_agent:?}"
3514 );
3515 }
3516 }
3517
3518 #[test]
3519 fn user_agent_with_token_appends_or_starts() {
3520 let token = expected_token();
3521 assert_eq!(
3522 user_agent_with_token(Some("aws-sdk-rust/1.3.0 os/linux")),
3523 format!("aws-sdk-rust/1.3.0 os/linux {token}")
3524 );
3525 assert_eq!(user_agent_with_token(None), token);
3526 }
3527
3528 fn is_version_number(s: &str) -> bool {
3530 !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) && (s == "0" || !s.starts_with('0'))
3531 }
3532
3533 fn has_supported_version_form(version: &str) -> bool {
3536 let (core, pre_release) = match version.split_once('-') {
3537 Some((core, pre_release)) => (core, Some(pre_release)),
3538 None => (version, None),
3539 };
3540 let numbers: Vec<&str> = core.split('.').collect();
3541 if numbers.len() != 3 || !numbers.iter().all(|n| is_version_number(n)) {
3542 return false;
3543 }
3544 let Some(pre_release) = pre_release else {
3545 return true;
3546 };
3547 let (label, number) = match pre_release.split_once('.') {
3548 Some((label, number)) => (label, Some(number)),
3549 None => (pre_release, None),
3550 };
3551 matches!(label, "alpha" | "beta" | "rc") && number.is_none_or(is_version_number)
3552 }
3553
3554 #[test]
3555 fn crate_version_has_the_supported_form() {
3556 for version in [
3557 "0.1.0",
3558 "1.0.0",
3559 "1.1.0-alpha",
3560 "1.1.0-beta.2",
3561 "1.1.0-rc.1",
3562 ] {
3563 assert!(has_supported_version_form(version), "{version} must fit");
3564 }
3565 for version in [
3566 "1.0.0-rc1",
3567 "1.0.0-preview",
3568 "1.0.0-RC.1",
3569 "1.0.0+build.1",
3570 "v1.0.0",
3571 "1.0",
3572 "01.0.0",
3573 "1.0.0-rc.01",
3574 ] {
3575 assert!(
3576 !has_supported_version_form(version),
3577 "{version} must not fit"
3578 );
3579 }
3580 assert!(
3581 has_supported_version_form(env!("CARGO_PKG_VERSION")),
3582 "the crate version {} has an unsupported form",
3583 env!("CARGO_PKG_VERSION")
3584 );
3585 }
3586}