Skip to main content

aws_durable_execution_sdk/
client.rs

1//! The checkpoint transport: the trait the SDK calls to checkpoint
2//! operation updates and read execution state, its error type, and the
3//! default transport backed by `aws-sdk-lambda`.
4
5use std::pin::Pin;
6use std::sync::Arc;
7#[cfg(test)]
8use std::sync::Mutex;
9
10use aws_sdk_lambda::types::{Operation, OperationType, OperationUpdate};
11
12use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
13
14/// The recovery scope of an [`ExecutionClientError`].
15///
16/// The scope decides what the SDK does with a transport failure. See the
17/// [`ExecutionClient`] contract for each case.
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19#[non_exhaustive]
20pub enum ExecutionClientErrorScope {
21    /// The invocation fails; the service re-invokes and replay resumes.
22    Invocation,
23    /// The execution fails.
24    Execution,
25}
26
27/// The terminal outcome a transport returns: a scope, a message and,
28/// when the transport has one, the error that caused it.
29///
30/// [`source()`](std::error::Error::source) returns the error passed to
31/// [`with_source`](Self::with_source).
32///
33/// A transport puts the detail a log needs into the message. The built-in
34/// [`LambdaExecutionClient`] keeps the full `DisplayErrorContext` text of
35/// the AWS SDK error in the message, and passes the final `SdkError` as the
36/// source.
37///
38/// The name of this type is not a wire `ErrorType`, and its `Display` text
39/// is not a wire `ErrorMessage`. An [`Execution`](ExecutionClientErrorScope::Execution)-scoped
40/// error that ends the bootstrap state read records the AWS error code when
41/// the Lambda transport supplied it, and `DurableExecutionClientError`
42/// otherwise. A recorded message is the AWS error's message when the Lambda
43/// transport supplied one, and the message passed to [`new`](Self::new) or
44/// [`with_source`](Self::with_source) otherwise.
45///
46/// `Clone` because a coalesced checkpoint batch publishes one result to
47/// every operation that contributed updates to it. The source is held in an
48/// `Arc` for that reason.
49#[derive(Debug, Clone)]
50#[non_exhaustive]
51pub struct ExecutionClientError {
52    scope: ExecutionClientErrorScope,
53    message: String,
54    source: Option<Arc<dyn std::error::Error + Send + Sync>>,
55}
56
57impl std::fmt::Display for ExecutionClientError {
58    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59        write!(f, "durable: client: {}", self.message)
60    }
61}
62
63impl std::error::Error for ExecutionClientError {
64    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
65        self.source
66            .as_deref()
67            .map(|s| s as &(dyn std::error::Error + 'static))
68    }
69}
70
71impl ExecutionClientError {
72    /// Creates an error with the given scope and message, and no source.
73    ///
74    /// # Examples
75    ///
76    /// ```
77    /// use aws_durable_execution_sdk::{ExecutionClientError, ExecutionClientErrorScope};
78    ///
79    /// let err = ExecutionClientError::new(ExecutionClientErrorScope::Execution, "rejected");
80    /// assert_eq!(err.scope(), ExecutionClientErrorScope::Execution);
81    /// assert_eq!(err.to_string(), "durable: client: rejected");
82    /// ```
83    #[must_use]
84    pub fn new(scope: ExecutionClientErrorScope, message: impl Into<String>) -> Self {
85        Self {
86            scope,
87            message: message.into(),
88            source: None,
89        }
90    }
91
92    /// Creates an error with the given scope and message, keeping `source`
93    /// as the error that caused it.
94    ///
95    /// # Examples
96    ///
97    /// ```
98    /// use std::error::Error as _;
99    /// use aws_durable_execution_sdk::{ExecutionClientError, ExecutionClientErrorScope};
100    ///
101    /// let cause = std::io::Error::other("connection reset");
102    /// let err = ExecutionClientError::with_source(
103    ///     ExecutionClientErrorScope::Invocation,
104    ///     "checkpoint failed",
105    ///     cause,
106    /// );
107    /// assert!(err.source().is_some());
108    /// ```
109    #[must_use]
110    pub fn with_source(
111        scope: ExecutionClientErrorScope,
112        message: impl Into<String>,
113        source: impl Into<crate::BoxError>,
114    ) -> Self {
115        Self {
116            scope,
117            message: message.into(),
118            source: Some(Arc::from(source.into())),
119        }
120    }
121
122    /// The recovery scope of this error.
123    pub fn scope(&self) -> ExecutionClientErrorScope {
124        self.scope
125    }
126
127    /// Whether this error fails only the invocation: `true` exactly for
128    /// [`ExecutionClientErrorScope::Invocation`]. `false` means a permanent
129    /// rejection, which fails the execution.
130    pub(crate) fn is_retryable(&self) -> bool {
131        self.scope == ExecutionClientErrorScope::Invocation
132    }
133
134    /// The error's message, without the `durable: client: ` prefix that
135    /// `Display` adds.
136    pub(crate) fn message(&self) -> &str {
137        &self.message
138    }
139
140    /// Creates a non-retryable error from an owned message. Only test
141    /// doubles call it: the Lambda transport builds every error with a
142    /// source.
143    #[cfg(test)]
144    pub(crate) fn non_retryable(message: String) -> Self {
145        Self::new(ExecutionClientErrorScope::Execution, message)
146    }
147
148    /// Creates a non-retryable error from a string slice (convenience).
149    pub(crate) fn new_non_retryable(message: &str) -> Self {
150        Self::new(ExecutionClientErrorScope::Execution, message)
151    }
152
153    /// Creates an `Invocation`-scoped error from an owned message. Only
154    /// test doubles call it: the Lambda transport builds every error with a
155    /// source.
156    #[cfg(any(test, feature = "test-util"))]
157    pub(crate) fn from_retryable(message: String) -> Self {
158        Self::new(ExecutionClientErrorScope::Invocation, message)
159    }
160}
161
162/// The message the SDK records for a transport error: the service's
163/// message when the Lambda transport supplied an AWS error, and the
164/// transport's own message otherwise.
165///
166/// It reads, in this order:
167///
168/// 1. The `message()` of the AWS error, when the source is the final
169///    `SdkError` of `CheckpointDurableExecution` or
170///    `GetDurableExecutionState` and that error has a message. JS reads
171///    `awsError.message` the same way.
172/// 2. Otherwise [`ExecutionClientError::message`], without the
173///    `durable: client: ` prefix of `Display`, as JS reads the `message` of
174///    the error the transport threw.
175pub(crate) fn client_error_record_message(err: &ExecutionClientError) -> String {
176    use aws_sdk_lambda::error::ProvideErrorMetadata;
177    use std::error::Error as _;
178
179    if let Some(source) = err.source() {
180        if let Some(sdk) = source.downcast_ref::<CheckpointSdkError>()
181            && let Some(m) = sdk.message()
182        {
183            return m.to_owned();
184        }
185        if let Some(sdk) = source.downcast_ref::<GetStateSdkError>()
186            && let Some(m) = sdk.message()
187        {
188            return m.to_owned();
189        }
190    }
191    err.message().to_owned()
192}
193
194/// The wire `ErrorType` of the bootstrap `FAILED` record for `err`.
195///
196/// When the source is the final `SdkError` of `GetDurableExecutionState`
197/// and it carries an AWS error code, the type is that code, as JS records
198/// the AWS error's `name`. Otherwise the type is
199/// [`EXECUTION_CLIENT_ERROR_TYPE`](crate::error::EXECUTION_CLIENT_ERROR_TYPE),
200/// as JS records for its `DurableExecutionClientError`.
201pub(crate) fn bootstrap_error_type(err: &ExecutionClientError) -> String {
202    use aws_sdk_lambda::error::ProvideErrorMetadata;
203    use std::error::Error as _;
204
205    err.source()
206        .and_then(|s| s.downcast_ref::<GetStateSdkError>())
207        .and_then(ProvideErrorMetadata::code)
208        .unwrap_or(crate::error::EXECUTION_CLIENT_ERROR_TYPE)
209        .to_owned()
210}
211
212/// The final error of a `CheckpointDurableExecution` call.
213type CheckpointSdkError = aws_sdk_lambda::error::SdkError<
214    aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError,
215    aws_sdk_lambda::config::http::HttpResponse,
216>;
217
218/// The final error of a `GetDurableExecutionState` call.
219type GetStateSdkError = aws_sdk_lambda::error::SdkError<
220    aws_sdk_lambda::operation::get_durable_execution_state::GetDurableExecutionStateError,
221    aws_sdk_lambda::config::http::HttpResponse,
222>;
223
224/// The output of a successful [`ExecutionClient::checkpoint`] call.
225#[derive(Debug, Clone)]
226#[non_exhaustive]
227pub struct CheckpointOutput {
228    /// The new checkpoint token for subsequent calls. `None` when the
229    /// service returned no token.
230    pub(crate) checkpoint_token: Option<String>,
231    /// Updated operations from the backend (may be empty).
232    pub(crate) updated_operations: Vec<Operation>,
233    /// Pagination marker: if present, more operations are available via
234    /// `get_state` and the caller must paginate.
235    pub(crate) next_marker: Option<String>,
236}
237
238impl CheckpointOutput {
239    /// Creates a checkpoint output.
240    ///
241    /// `checkpoint_token` is the token for the next request, or `None` when
242    /// the service returned none. `next_marker` is the pagination marker of
243    /// the returned state, or `None` when `updated_operations` is complete.
244    #[must_use]
245    pub fn new(
246        checkpoint_token: Option<String>,
247        updated_operations: Vec<Operation>,
248        next_marker: Option<String>,
249    ) -> Self {
250        Self {
251            checkpoint_token,
252            updated_operations,
253            next_marker,
254        }
255    }
256
257    /// The token for the next request. An empty token reads as `None`.
258    pub fn checkpoint_token(&self) -> Option<&str> {
259        self.checkpoint_token.as_deref().filter(|t| !t.is_empty())
260    }
261}
262
263/// One page of execution state, returned by [`ExecutionClient::get_state`].
264#[derive(Debug, Clone)]
265#[non_exhaustive]
266pub struct GetStateOutput {
267    /// The operations of this page.
268    pub(crate) operations: Vec<Operation>,
269    /// The marker of the next page, `None` on the last page.
270    pub(crate) next_marker: Option<String>,
271}
272
273impl GetStateOutput {
274    /// Creates one page of execution state. `next_marker` is the marker of
275    /// the next page, or `None` on the last page.
276    #[must_use]
277    pub fn new(operations: Vec<Operation>, next_marker: Option<String>) -> Self {
278        Self {
279            operations,
280            next_marker,
281        }
282    }
283
284    /// The marker of the next page. An empty marker reads as `None`.
285    pub fn next_marker(&self) -> Option<&str> {
286        self.next_marker.as_deref().filter(|m| !m.is_empty())
287    }
288}
289
290/// The transport for the two durable execution service calls.
291///
292/// The SDK uses [`LambdaExecutionClient`] unless
293/// [`OptionsBuilder::execution_client`](crate::OptionsBuilder::execution_client)
294/// supplies another implementation. The trait is object-safe, so a
295/// transport is passed as `Arc<dyn ExecutionClient>`.
296///
297/// # Contract
298///
299/// 1. [`checkpoint`](Self::checkpoint) sends `updates` in order in one
300///    request. It returns the new token, the operations the backend
301///    returned, and the pagination marker of the returned state.
302/// 2. The returned token is `None` when the service accepted the request
303///    and returned no token for the next one. An empty token is reported
304///    as `None`.
305/// 3. [`get_state`](Self::get_state) returns one page of the execution's
306///    operations and the marker of the next page, or `None` on the last
307///    page. The SDK follows the marker.
308/// 4. Transient retry is the transport's job. An error the method returns
309///    is final for the invocation.
310/// 5. An error's [`scope`](ExecutionClientError::scope) decides the
311///    outcome:
312///    - [`Invocation`](ExecutionClientErrorScope::Invocation) fails the
313///      invocation; the service re-invokes and replay resumes.
314///    - [`Execution`](ExecutionClientErrorScope::Execution) from
315///      `checkpoint` fails the execution. The SDK sends no further
316///      checkpoint in the invocation and writes no further update for the
317///      operation, so its record stays at the last update the service
318///      accepted. The execution's `FAILED` record has `ErrorType`
319///      `CheckpointUnrecoverableExecutionError` and `ErrorMessage`
320///      `[Unrecoverable Execution] Checkpoint failed: {m}`.
321///      `{m}` is the AWS error's message when the Lambda transport supplied
322///      one, and the error's own message otherwise.
323///    - `Execution` from the bootstrap `get_state` returns a `FAILED`
324///      response. Its `ErrorType` is the AWS error code when the Lambda
325///      transport supplied one, and `DurableExecutionClientError`
326///      otherwise. Its `ErrorMessage` is `{m}` as above.
327pub trait ExecutionClient: Send + Sync {
328    /// Checkpoints operation updates and returns the next token.
329    fn checkpoint(
330        &self,
331        execution_arn: &str,
332        checkpoint_token: &str,
333        updates: Vec<OperationUpdate>,
334    ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>;
335
336    /// Reads one page of the execution state. `None` reads the first page,
337    /// and `Some(m)` reads the page after marker `m`.
338    fn get_state(
339        &self,
340        execution_arn: &str,
341        checkpoint_token: &str,
342        marker: Option<&str>,
343    ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>;
344}
345
346/// The page size of a `GetDurableExecutionState` request, as in JS.
347const GET_STATE_MAX_ITEMS: i32 = 1000;
348
349/// The token the SDK adds to the `User-Agent` header of its own requests.
350///
351/// The name is a literal because the crate is named
352/// `aws-durable-execution-sdk`, and the durable execution SDKs for the other
353/// languages send `aws-durable-execution-sdk-<language>/<version>`. The
354/// version is the crate version, so the token always matches it.
355const USER_AGENT_TOKEN: &str =
356    concat!("aws-durable-execution-sdk-rust/", env!("CARGO_PKG_VERSION"));
357
358/// Appends [`USER_AGENT_TOKEN`] to the `User-Agent` header of one request.
359///
360/// The AWS SDK writes `User-Agent` in `modify_before_signing`. This
361/// interceptor runs later, in `modify_before_transmit`. `SigV4` leaves
362/// `user-agent` out of the signature, so the change keeps the signature
363/// valid. The runtime rewinds the request before each retry attempt, so
364/// each attempt appends the token once to a freshly written header.
365///
366/// `framework_metadata` on the client config is not used, because it
367/// writes the token with a `lib/` prefix.
368#[derive(Debug)]
369struct UserAgentTokenInterceptor;
370
371impl aws_sdk_lambda::config::Intercept for UserAgentTokenInterceptor {
372    fn name(&self) -> &'static str {
373        "UserAgentTokenInterceptor"
374    }
375
376    fn modify_before_transmit(
377        &self,
378        context: &mut aws_sdk_lambda::config::interceptors::BeforeTransmitInterceptorContextMut<'_>,
379        _runtime_components: &aws_sdk_lambda::config::RuntimeComponents,
380        _cfg: &mut aws_sdk_lambda::config::ConfigBag,
381    ) -> Result<(), aws_sdk_lambda::error::BoxError> {
382        let headers = context.request_mut().headers_mut();
383        let value = user_agent_with_token(headers.get("user-agent"));
384        // `insert` replaces the value. `append` would add a second
385        // `User-Agent` header. The value is the current valid header value,
386        // a space and an ASCII token, so it is a valid header value.
387        headers.insert("user-agent", value);
388        Ok(())
389    }
390}
391
392/// Returns `current` followed by a space and [`USER_AGENT_TOKEN`], or the
393/// token alone when the request has no `User-Agent` header. The AWS SDK
394/// always writes the header, but an interceptor on a caller's client can
395/// remove it.
396fn user_agent_with_token(current: Option<&str>) -> String {
397    match current {
398        Some(current) => format!("{current} {USER_AGENT_TOKEN}"),
399        None => USER_AGENT_TOKEN.to_owned(),
400    }
401}
402
403/// The transport the SDK uses when `execution_client` is not set. Wrap it
404/// to decorate the default.
405///
406/// Retry is delegated entirely to the wrapped SDK client's own retry
407/// strategy (standard retry by default: jittered exponential backoff, 3
408/// attempts). This matches the sibling JS and Python SDKs, which rely on
409/// their AWS SDKs' built-in retry. A call that exhausts the SDK's retries
410/// fails the invocation; the durable execution service re-invokes the
411/// handler, which is the recovery path: longer in-process retry would
412/// only delay it. Callers who need a different retry policy supply a
413/// preconfigured client.
414///
415/// Every request it sends carries the token
416/// `aws-durable-execution-sdk-rust/<version>` as one more space-separated
417/// part of its `User-Agent` header, where `<version>` is the crate version.
418/// The token is added to each request, so the wrapped client is not changed.
419///
420/// Every final error carries the final `SdkError` as its
421/// [`source`](std::error::Error::source).
422///
423/// # Examples
424///
425/// ```no_run
426/// use std::sync::Arc;
427/// use aws_durable_execution_sdk::{
428///     ExecutionClient, LambdaClient, LambdaExecutionClient, Options, SdkConfig,
429/// };
430///
431/// let client = LambdaClient::new(&SdkConfig::builder().build());
432/// let transport: Arc<dyn ExecutionClient> = Arc::new(LambdaExecutionClient::new(client));
433/// let options = Options::builder()
434///     .execution_client(transport)
435///     .build()
436///     .expect("valid config");
437/// # drop(options);
438/// ```
439#[derive(Clone)]
440#[non_exhaustive]
441pub struct LambdaExecutionClient {
442    client: aws_sdk_lambda::Client,
443}
444
445impl std::fmt::Debug for LambdaExecutionClient {
446    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
447        f.write_str("LambdaExecutionClient")
448    }
449}
450
451impl LambdaExecutionClient {
452    /// Creates a transport that calls the service through `client`.
453    #[must_use]
454    pub fn new(client: crate::LambdaClient) -> Self {
455        Self { client }
456    }
457}
458
459impl ExecutionClient for LambdaExecutionClient {
460    fn checkpoint(
461        &self,
462        execution_arn: &str,
463        checkpoint_token: &str,
464        updates: Vec<OperationUpdate>,
465    ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
466    {
467        let arn = execution_arn.to_owned();
468        let token = checkpoint_token.to_owned();
469        Box::pin(async move {
470            let result = self
471                .client
472                .checkpoint_durable_execution()
473                .durable_execution_arn(&arn)
474                .checkpoint_token(&token)
475                .set_updates(Some(updates))
476                .customize()
477                .interceptor(UserAgentTokenInterceptor)
478                .send()
479                .await;
480
481            match result {
482                Ok(output) => {
483                    let new_token = output.checkpoint_token.filter(|t| !t.is_empty());
484                    let (updated_ops, next_marker) = match output.new_execution_state {
485                        Some(state) => (
486                            state.operations.unwrap_or_default(),
487                            state.next_marker.filter(|m| !m.is_empty()),
488                        ),
489                        None => (Vec::new(), None),
490                    };
491                    Ok(CheckpointOutput::new(new_token, updated_ops, next_marker))
492                }
493                // The SDK's standard retry has already retried everything
494                // transient; the final error is classified into a recovery
495                // scope (invocation vs execution): see
496                // `classify_checkpoint_error`.
497                Err(err) => Err(classify_checkpoint_error(err)),
498            }
499        })
500    }
501
502    /// Sends one `GetDurableExecutionState` request with `MaxItems` 1000.
503    /// The four KMS errors are `Execution`-scoped, and every other final
504    /// error, a plain 4xx included, is `Invocation`-scoped, as in JS.
505    fn get_state(
506        &self,
507        execution_arn: &str,
508        checkpoint_token: &str,
509        marker: Option<&str>,
510    ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
511    {
512        let arn = execution_arn.to_owned();
513        let token = checkpoint_token.to_owned();
514        let marker = marker.map(str::to_owned);
515        Box::pin(async move {
516            // As with `checkpoint`, the SDK's standard retry owns the
517            // transient-failure retry; the final error is classified by
518            // `classify_get_state_error`.
519            let output = self
520                .client
521                .get_durable_execution_state()
522                .durable_execution_arn(&arn)
523                .checkpoint_token(&token)
524                .set_marker(marker)
525                .max_items(GET_STATE_MAX_ITEMS)
526                .customize()
527                .interceptor(UserAgentTokenInterceptor)
528                .send()
529                .await
530                .map_err(classify_get_state_error)?;
531
532            Ok(GetStateOutput::new(
533                output.operations,
534                output.next_marker.filter(|m| !m.is_empty()),
535            ))
536        })
537    }
538}
539
540/// Reads the whole execution state: calls [`ExecutionClient::get_state`]
541/// with no marker, then with each returned marker until
542/// [`GetStateOutput::next_marker`] is `None`, and concatenates the pages.
543pub(crate) async fn get_full_state(
544    client: &dyn ExecutionClient,
545    execution_arn: &str,
546    checkpoint_token: &str,
547) -> Result<Vec<Operation>, ExecutionClientError> {
548    let mut operations = Vec::new();
549    let mut marker: Option<String> = None;
550    loop {
551        let page = client
552            .get_state(execution_arn, checkpoint_token, marker.as_deref())
553            .await?;
554        let next = page.next_marker().map(str::to_owned);
555        operations.extend(page.operations);
556        match next {
557            Some(m) => marker = Some(m),
558            None => return Ok(operations),
559        }
560    }
561}
562
563/// The message prefix the service puts on an `InvalidParameterValueException`
564/// that rejects a stale checkpoint token.
565///
566/// A stale token resolves on re-invocation (the service issues a fresh one),
567/// so this specific rejection is classified retryable: the carve-out the
568/// Java SDK's `DurableApiErrorClassifier` applies.
569const STALE_TOKEN_MESSAGE_PREFIX: &str = "Invalid checkpoint token";
570
571/// Classifies a final `CheckpointDurableExecution` error into a recovery
572/// scope, carried as [`ExecutionClientError::scope`]. The final error is
573/// kept as the source.
574///
575/// By the time this runs, the aws-sdk's standard retry has exhausted its
576/// transport-level attempts, so the question is no longer "try again now"
577/// but *which recovery converges*:
578///
579/// - **Retryable**: fail the invocation; the durable service re-invokes
580///   and replay resumes from the recorded state. This is the same recovery
581///   as an interruption. Applied to transport-level failures (dispatch,
582///   timeout, a malformed response without a 4xx status), throttling (any
583///   429), 5xx, and the stale-token carve-out
584///   ([`STALE_TOKEN_MESSAGE_PREFIX`]).
585/// - **Non-retryable**: the service permanently rejected this write (for
586///   example an oversized payload, a KMS misconfiguration, or any other
587///   4xx except 429). Re-invoking re-runs the body and hits
588///   the same rejection, so the SDK fails the execution instead.
589///
590/// After the modeled variants, the classifier reads the HTTP status of the
591/// raw response, as JS reads `$metadata.httpStatusCode`. A status from 400
592/// to 499 other than 429 is non-retryable: the service rejected the
593/// request, so the same write fails the same way on every invocation. This
594/// covers an unmodeled service error, such as a 403
595/// `AccessDeniedException` when the role lacks
596/// `lambda:CheckpointDurableExecution`, and a response the SDK could not
597/// parse. Every other status, and an error without a raw response
598/// (dispatch failure, timeout, construction failure), is retryable.
599///
600/// **An unmodeled error is retryable unless its status is a 4xx other
601/// than 429.** The original classification
602/// (pre-#43) defaulted unknown to non-retryable, with the rationale, from
603/// the r3726032098 review thread, that the surfaced error only failed the
604/// invocation and the service re-invoked anyway, so the default was
605/// harmless. That rationale belongs to the old model: under this model a
606/// non-retryable classification fails the *execution*, so a novel
607/// transient error variant defaulting to non-retryable would kill
608/// executions. A novel transient error arrives as 429, 5xx or a transport
609/// failure, and those stay retryable; a 4xx other than 429 is a permanent
610/// rejection, which JS also fails the execution on. Java and Python
611/// default unknown to retryable for the same reason. The
612/// `checkpoint_error_variant_canary` test pins the variant set this
613/// function classifies explicitly, so a new upstream variant fails CI for
614/// reclassification instead of falling silently into the status rule.
615fn classify_checkpoint_error(err: CheckpointSdkError) -> ExecutionClientError {
616    let message = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(&err));
617    let scope = checkpoint_error_scope(&err);
618    ExecutionClientError::with_source(scope, message, err)
619}
620
621/// The recovery scope of a final `CheckpointDurableExecution` error: see
622/// [`classify_checkpoint_error`].
623fn checkpoint_error_scope(err: &CheckpointSdkError) -> ExecutionClientErrorScope {
624    use ExecutionClientErrorScope::{Execution, Invocation};
625    use aws_sdk_lambda::error::SdkError;
626    use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError as E;
627
628    if let SdkError::ServiceError(service_err) = err {
629        match service_err.err() {
630            // Throttling and service-side 5xx: transient by definition.
631            E::TooManyRequestsException(_) | E::ServiceException(_) => return Invocation,
632            // A parameter rejection is deterministic, the same write fails
633            // the same way on every lap, EXCEPT the stale-token rejection,
634            // which a re-invocation resolves with a fresh token.
635            E::InvalidParameterValueException(inner) => {
636                return if inner
637                    .message()
638                    .is_some_and(|m| m.starts_with(STALE_TOKEN_MESSAGE_PREFIX))
639                {
640                    Invocation
641                } else {
642                    Execution
643                };
644            }
645            // KMS misconfiguration on the function: deterministic until an
646            // operator fixes the key, so re-invoking loops without progress.
647            E::KmsAccessDeniedException(_)
648            | E::KmsDisabledException(_)
649            | E::KmsInvalidStateException(_)
650            | E::KmsNotFoundException(_) => return Execution,
651            // Unmodeled errors fall through to the status rule. The variant
652            // canary test keeps this arm from silently absorbing new modeled
653            // variants.
654            _ => {}
655        }
656    }
657
658    // The status rule: see the function docs. Errors without a raw response
659    // (dispatch failure, timeout, construction failure) are channel-level:
660    // nothing was rejected.
661    match err.raw_response().map(|r| r.status().as_u16()) {
662        Some(status) if (400..500).contains(&status) && status != 429 => Execution,
663        _ => Invocation,
664    }
665}
666
667/// Classifies a final `GetDurableExecutionState` error into a recovery
668/// scope, carried as [`ExecutionClientError::scope`]. The final error is
669/// kept as the source.
670///
671/// The four KMS errors are non-retryable: the function's key is
672/// misconfigured, so every re-invocation fails the same way. Every other
673/// error is retryable, a plain 4xx included, because JS rethrows every
674/// other error of its bootstrap state read and so fails only the
675/// invocation. The message is the `DisplayErrorContext` text, so it names
676/// the exception.
677fn classify_get_state_error(err: GetStateSdkError) -> ExecutionClientError {
678    use aws_sdk_lambda::operation::get_durable_execution_state::GetDurableExecutionStateError as E;
679
680    let message = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(&err));
681    let scope = match err.as_service_error() {
682        Some(
683            E::KmsAccessDeniedException(_)
684            | E::KmsDisabledException(_)
685            | E::KmsInvalidStateException(_)
686            | E::KmsNotFoundException(_),
687        ) => ExecutionClientErrorScope::Execution,
688        _ => ExecutionClientErrorScope::Invocation,
689    };
690    ExecutionClientError::with_source(scope, message, err)
691}
692
693/// An HTTP error response as the service sends it: the AWS error code in
694/// the `x-amzn-errortype` header and the message in the JSON body. The
695/// generated deserializer reads both into the error's metadata, which a
696/// modeled error built in a test does not carry.
697#[cfg(test)]
698pub(crate) fn mock_aws_error_response(
699    status: u16,
700    code: &str,
701    message: &str,
702) -> aws_sdk_lambda::config::http::HttpResponse {
703    let body = serde_json::json!({ "Type": "User", "message": message }).to_string();
704    let mut response = aws_sdk_lambda::config::http::HttpResponse::new(
705        aws_smithy_runtime_api::http::StatusCode::try_from(status).expect("valid HTTP status"),
706        aws_smithy_types::body::SdkBody::from(body),
707    );
708    response
709        .headers_mut()
710        .insert("x-amzn-errortype", code.to_owned());
711    response
712        .headers_mut()
713        .insert("content-type", "application/json");
714    response
715}
716
717/// Injection point for controlling test double behavior per call.
718#[cfg(test)]
719#[derive(Debug, Clone)]
720pub(crate) enum TestResponse {
721    /// Return success with the given operations.
722    Success(Vec<Operation>),
723    /// Return success with given operations and a pagination marker,
724    /// indicating more operations are available via `get_state`.
725    SuccessPaginated(Vec<Operation>, String),
726    /// Return a retryable failure.
727    RetryableError(String),
728    /// Return a non-retryable failure.
729    NonRetryableError(String),
730    /// Accept the request's updates, then return a retryable failure, as
731    /// when the service recorded a write whose response was lost.
732    AcceptThenRetryableError(String),
733}
734
735/// In-memory test double for `ExecutionClient`.
736///
737/// Supports injecting failures and recording calls for assertions.
738/// Sufficient for unit tests, not the full `test-util` `LocalRunner`.
739#[cfg(test)]
740#[derive(Debug)]
741pub(crate) struct InMemoryExecutionClient {
742    /// Pre-loaded state returned by `get_state`.
743    state_operations: Mutex<Vec<Operation>>,
744    /// Queue of responses for checkpoint calls; if empty, returns success.
745    checkpoint_responses: Mutex<Vec<TestResponse>>,
746    /// Counter for checkpoint calls made (including retries).
747    pub(crate) checkpoint_call_count: Mutex<u32>,
748    /// Counter for `get_state` calls made.
749    pub(crate) get_state_call_count: Mutex<u32>,
750    /// When set, `get_state` fails with a non-retryable error carrying this
751    /// message (used to test that a persisted checkpoint's lifecycle events
752    /// survive a failed pagination fetch).
753    get_state_failure: Mutex<Option<String>>,
754    /// Token counter for generating unique tokens.
755    token_counter: Mutex<u32>,
756    /// Counter for backend-style callback ID assignment on Callback START.
757    callback_counter: Mutex<u32>,
758    /// All operation updates received across checkpoint calls.
759    recorded_updates: Mutex<Vec<OperationUpdate>>,
760}
761
762#[cfg(test)]
763impl InMemoryExecutionClient {
764    /// Creates a new test double with the given pre-loaded state.
765    pub(crate) fn new(state_operations: Vec<Operation>) -> Self {
766        Self {
767            state_operations: Mutex::new(state_operations),
768            checkpoint_responses: Mutex::new(Vec::new()),
769            checkpoint_call_count: Mutex::new(0),
770            get_state_call_count: Mutex::new(0),
771            get_state_failure: Mutex::new(None),
772            token_counter: Mutex::new(0),
773            callback_counter: Mutex::new(0),
774            recorded_updates: Mutex::new(Vec::new()),
775        }
776    }
777
778    /// Enqueues a response for the next checkpoint call.
779    pub(crate) fn enqueue_checkpoint_response(&self, response: TestResponse) {
780        let mut responses = self
781            .checkpoint_responses
782            .lock()
783            .unwrap_or_else(std::sync::PoisonError::into_inner);
784        responses.push(response);
785    }
786
787    /// Makes every subsequent `get_state` call fail non-retryably with the
788    /// given message.
789    pub(crate) fn fail_get_state(&self, message: &str) {
790        let mut failure = self
791            .get_state_failure
792            .lock()
793            .unwrap_or_else(std::sync::PoisonError::into_inner);
794        *failure = Some(message.to_owned());
795    }
796
797    /// Returns all operation updates recorded across checkpoint calls.
798    pub(crate) fn recorded_updates(&self) -> Vec<OperationUpdate> {
799        self.recorded_updates
800            .lock()
801            .unwrap_or_else(std::sync::PoisonError::into_inner)
802            .clone()
803    }
804}
805
806#[cfg(test)]
807impl ExecutionClient for InMemoryExecutionClient {
808    fn checkpoint(
809        &self,
810        _execution_arn: &str,
811        _checkpoint_token: &str,
812        updates: Vec<OperationUpdate>,
813    ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
814    {
815        {
816            let mut count = self
817                .checkpoint_call_count
818                .lock()
819                .unwrap_or_else(std::sync::PoisonError::into_inner);
820            *count += 1;
821        }
822
823        // Mirror the backend contract: the Callback START response carries
824        // the backend-assigned callback ID (issue #47). Assign one per
825        // Callback START update so the default success path behaves like
826        // the real service; explicitly enqueued responses stay
827        // authoritative and receive no auto-assigned operations.
828        let auto_ops: Vec<Operation> = updates
829            .iter()
830            .filter(|u| {
831                u.r#type() == &OperationType::Callback
832                    && u.action() == &aws_sdk_lambda::types::OperationAction::Start
833            })
834            .map(|u| {
835                let mut counter = self
836                    .callback_counter
837                    .lock()
838                    .unwrap_or_else(std::sync::PoisonError::into_inner);
839                *counter += 1;
840                Operation::builder()
841                    .id(u.id())
842                    .r#type(OperationType::Callback)
843                    .status(aws_sdk_lambda::types::OperationStatus::Pending)
844                    .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
845                    .callback_details(
846                        aws_sdk_lambda::types::CallbackDetails::builder()
847                            .callback_id(format!("in-mem-cb-{counter}"))
848                            .build(),
849                    )
850                    .build()
851                    .expect("all required Operation fields set")
852            })
853            .collect();
854
855        // Record the updates for test assertions.
856        {
857            let mut recorded = self
858                .recorded_updates
859                .lock()
860                .unwrap_or_else(std::sync::PoisonError::into_inner);
861            recorded.extend(updates);
862        }
863
864        let response = {
865            let mut responses = self
866                .checkpoint_responses
867                .lock()
868                .unwrap_or_else(std::sync::PoisonError::into_inner);
869            if responses.is_empty() {
870                None
871            } else {
872                Some(responses.remove(0))
873            }
874        };
875
876        Box::pin(async move {
877            match response {
878                // The updates are recorded above in both cases.
879                Some(
880                    TestResponse::RetryableError(msg) | TestResponse::AcceptThenRetryableError(msg),
881                ) => Err(ExecutionClientError::from_retryable(msg)),
882                Some(TestResponse::NonRetryableError(msg)) => {
883                    Err(ExecutionClientError::non_retryable(msg))
884                }
885                Some(TestResponse::Success(ops)) => {
886                    let mut counter = self
887                        .token_counter
888                        .lock()
889                        .unwrap_or_else(std::sync::PoisonError::into_inner);
890                    *counter += 1;
891                    Ok(CheckpointOutput {
892                        checkpoint_token: Some(format!("token-{counter}")),
893                        updated_operations: ops,
894                        next_marker: None,
895                    })
896                }
897                Some(TestResponse::SuccessPaginated(ops, marker)) => {
898                    let mut counter = self
899                        .token_counter
900                        .lock()
901                        .unwrap_or_else(std::sync::PoisonError::into_inner);
902                    *counter += 1;
903                    Ok(CheckpointOutput {
904                        checkpoint_token: Some(format!("token-{counter}")),
905                        updated_operations: ops,
906                        next_marker: Some(marker),
907                    })
908                }
909                None => {
910                    let mut counter = self
911                        .token_counter
912                        .lock()
913                        .unwrap_or_else(std::sync::PoisonError::into_inner);
914                    *counter += 1;
915                    Ok(CheckpointOutput {
916                        checkpoint_token: Some(format!("token-{counter}")),
917                        updated_operations: auto_ops,
918                        next_marker: None,
919                    })
920                }
921            }
922        })
923    }
924
925    fn get_state(
926        &self,
927        _execution_arn: &str,
928        _checkpoint_token: &str,
929        _marker: Option<&str>,
930    ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
931    {
932        {
933            let mut count = self
934                .get_state_call_count
935                .lock()
936                .unwrap_or_else(std::sync::PoisonError::into_inner);
937            *count += 1;
938        }
939
940        let failure = self
941            .get_state_failure
942            .lock()
943            .unwrap_or_else(std::sync::PoisonError::into_inner)
944            .clone();
945        if let Some(message) = failure {
946            return Box::pin(async move { Err(ExecutionClientError::non_retryable(message)) });
947        }
948
949        let ops = self
950            .state_operations
951            .lock()
952            .unwrap_or_else(std::sync::PoisonError::into_inner)
953            .clone();
954
955        Box::pin(async move { Ok(GetStateOutput::new(ops, None)) })
956    }
957}
958
959/// Converts a single SDK `Operation` into a `(wire_id, CheckpointRecord)` pair.
960fn operation_to_record(op: &Operation) -> (String, CheckpointRecord) {
961    let attempt = op.step_details.as_ref().map_or(0, |s| {
962        #[expect(clippy::cast_sign_loss)] // reason: attempt is non-negative from backend
963        {
964            s.attempt.max(0) as u32
965        }
966    });
967    let record = CheckpointRecord {
968        id: op.id().to_owned(),
969        status: sdk_status_to_checkpoint_status(op.status()),
970        result: extract_result(op),
971        error_type: extract_error_type(op),
972        error_message: extract_error_message(op),
973        error_data: extract_wire_error_field(op, |e| e.error_data.clone()),
974        stack_trace: extract_wire_error_field(op, |e| {
975            (!e.stack_trace().is_empty()).then(|| e.stack_trace().to_vec())
976        }),
977        attempt,
978        // ChainedInvoke payloads live in their own record fields: mirroring
979        // parse_single_operation in lib.rs, which is the JSON-path reference
980        // for this mapping. invoke.rs replays exclusively from these.
981        invoke_result: op
982            .chained_invoke_details
983            .as_ref()
984            .and_then(|i| i.result.clone()),
985        invoke_error_type: op
986            .chained_invoke_details
987            .as_ref()
988            .and_then(|i| i.error.as_ref())
989            .and_then(|e| e.error_type.clone()),
990        invoke_error_message: op
991            .chained_invoke_details
992            .as_ref()
993            .and_then(|i| i.error.as_ref())
994            .and_then(|e| e.error_message.clone()),
995        invoke_error_data: op
996            .chained_invoke_details
997            .as_ref()
998            .and_then(|i| i.error.as_ref())
999            .and_then(|e| e.error_data.clone()),
1000        invoke_stack_trace: op
1001            .chained_invoke_details
1002            .as_ref()
1003            .and_then(|i| i.error.as_ref())
1004            .and_then(|e| (!e.stack_trace().is_empty()).then(|| e.stack_trace().to_vec())),
1005        // Large child contexts require ContextDetails.ReplayChildren to
1006        // signal re-execution of children; mirror the JSON path.
1007        replay_children: op
1008            .context_details
1009            .as_ref()
1010            .and_then(|c| c.replay_children)
1011            .unwrap_or(false),
1012        callback_id: op
1013            .callback_details
1014            .as_ref()
1015            .and_then(|cb| cb.callback_id.clone()),
1016        op_type: Some(operation_type_to_string(&op.r#type)),
1017        sub_type: op.sub_type.clone(),
1018        op_name: op.name.clone(),
1019        due_at: operation_due_at(op),
1020        parent_id: op.parent_id().map(str::to_owned),
1021        start_timestamp: crate::to_system_time(*op.start_timestamp()),
1022        end_timestamp: op.end_timestamp().and_then(|t| crate::to_system_time(*t)),
1023    };
1024    (op.id().to_owned(), record)
1025}
1026
1027/// When `op` is next due: a WAIT's scheduled end, or a STEP's next attempt.
1028/// `None` for any other type, a missing value, or a value outside the range
1029/// of `SystemTime`.
1030fn operation_due_at(op: &Operation) -> Option<std::time::SystemTime> {
1031    let timestamp = match op.r#type() {
1032        OperationType::Wait => op
1033            .wait_details
1034            .as_ref()
1035            .and_then(|d| d.scheduled_end_timestamp),
1036        OperationType::Step => op
1037            .step_details
1038            .as_ref()
1039            .and_then(|d| d.next_attempt_timestamp),
1040        _ => None,
1041    }?;
1042    crate::to_system_time(timestamp)
1043}
1044
1045/// Returns the wire token of an `OperationType`, such as `CONTEXT`, for
1046/// every variant, including one this SDK version does not know.
1047fn operation_type_to_string(op_type: &OperationType) -> String {
1048    op_type.as_str().to_owned()
1049}
1050
1051/// Builds the plugin info of a fetched `EXECUTION` operation, or returns
1052/// `None` for an operation of any other type.
1053pub(crate) fn execution_info_from_operation(
1054    op: &Operation,
1055) -> Option<crate::plugin::OperationInfo> {
1056    crate::plugin::OperationInfo::execution(
1057        op.id(),
1058        op.r#type().as_str(),
1059        &sdk_status_to_checkpoint_status(op.status()),
1060        op.parent_id().map(str::to_owned),
1061        crate::to_system_time(*op.start_timestamp()),
1062        op.end_timestamp().and_then(|t| crate::to_system_time(*t)),
1063    )
1064}
1065
1066/// Converts SDK `Operation` records into a `CheckpointLog`.
1067///
1068/// Keyed by wire ID (the operation's `id` field from the service). Only
1069/// tests build a log this way: bootstrap merges the fetched pages into the
1070/// inline log in [`resolve_bootstrap_log`].
1071#[cfg(test)]
1072pub(crate) fn operations_to_checkpoint_log(operations: &[Operation]) -> CheckpointLog {
1073    let records: Vec<(String, CheckpointRecord)> = operations
1074        .iter()
1075        // Skip Execution-type operations for consistency with
1076        // parse_inline_operations, which filters them in the JSON path.
1077        .filter(|op| op.r#type != OperationType::Execution)
1078        .map(operation_to_record)
1079        .collect();
1080    CheckpointLog::from_records(records)
1081}
1082
1083/// Merges updated operations from a checkpoint response into an existing log.
1084///
1085/// After a checkpoint call, the backend may return operations with
1086/// backend-assigned fields (e.g. `callback_id`). These must be visible to
1087/// subsequent reads in the same invocation.
1088///
1089/// Skips every `EXECUTION` operation: the log holds none, and the plugins
1090/// keep the `EXECUTION` operation in their own slot. Returns the wire ID of
1091/// each merged operation whose status changed, in input order: the log
1092/// held no record of it, or held one with another status. Other fields do
1093/// not count.
1094pub(crate) fn merge_operations_into_log(
1095    log: &CheckpointLog,
1096    operations: &[Operation],
1097) -> Vec<String> {
1098    let mut changed = Vec::new();
1099    for op in operations {
1100        if op.r#type == OperationType::Execution {
1101            continue;
1102        }
1103        let (wire_id, record) = operation_to_record(op);
1104        let status = record.status.clone();
1105        if log.insert(wire_id.clone(), record).as_ref() != Some(&status) {
1106            changed.push(wire_id);
1107        }
1108    }
1109    changed
1110}
1111
1112/// The execution state the bootstrap reads: the checkpoint log, the
1113/// execution input and the `EXECUTION` operation.
1114#[derive(Debug)]
1115pub(crate) struct BootstrapState {
1116    /// The checkpoint log, without the `EXECUTION` operation.
1117    pub(crate) log: CheckpointLog,
1118    /// The `InputPayload` of the `EXECUTION` operation, or `None` when it
1119    /// is absent or no page holds the operation.
1120    pub(crate) input_payload: Option<String>,
1121    /// The plugin info of the `EXECUTION` operation, or `None` when no page
1122    /// read so far holds it.
1123    pub(crate) execution: Option<crate::plugin::OperationInfo>,
1124}
1125
1126/// Resolves the bootstrap state, following the pagination marker.
1127///
1128/// This is the single production decision point for bootstrap pagination.
1129/// `first_page` is the state parsed from the inline
1130/// `InitialExecutionState`, and `next_marker` is its `NextMarker`.
1131/// `first_page.execution.is_some()` says that the inline page holds the
1132/// `EXECUTION` operation, so its input is `first_page.input_payload`.
1133///
1134/// - Without a marker the inline page is the whole state. The function
1135///   returns `first_page` unchanged, and calls nothing.
1136/// - With a marker the function keeps `first_page`. It calls
1137///   [`ExecutionClient::get_state`] with the inline marker, merges each
1138///   returned page into the log, and calls again with each returned marker
1139///   until a page has none. It never reads the first page again.
1140///
1141/// When the inline page lacks the `EXECUTION` operation, the function takes
1142/// the operation and its `InputPayload` from the first `EXECUTION`
1143/// operation of the fetched pages, in page order. The caller parses the
1144/// input after this function returns, so an input on a later page reaches
1145/// the handler.
1146pub(crate) async fn resolve_bootstrap_log(
1147    client: &dyn ExecutionClient,
1148    execution_arn: &str,
1149    checkpoint_token: &str,
1150    first_page: BootstrapState,
1151    next_marker: Option<&str>,
1152) -> Result<BootstrapState, ExecutionClientError> {
1153    let mut state = first_page;
1154    let mut marker = next_marker.map(str::to_owned);
1155    while let Some(current) = marker {
1156        let page = client
1157            .get_state(execution_arn, checkpoint_token, Some(&current))
1158            .await?;
1159        marker = page.next_marker().map(str::to_owned);
1160        // The checkpoint log skips EXECUTION operations, so the input and
1161        // the operation are read from the raw page before it is merged.
1162        if state.execution.is_none()
1163            && let Some(op) = page
1164                .operations
1165                .iter()
1166                .find(|op| op.r#type == OperationType::Execution)
1167        {
1168            state.input_payload = op
1169                .execution_details
1170                .as_ref()
1171                .and_then(|d| d.input_payload.clone());
1172            state.execution = execution_info_from_operation(op);
1173        }
1174        for op in page
1175            .operations
1176            .iter()
1177            .filter(|op| op.r#type != OperationType::Execution)
1178        {
1179            let (wire_id, record) = operation_to_record(op);
1180            state.log.insert(wire_id, record);
1181        }
1182    }
1183    Ok(state)
1184}
1185
1186/// What one page of the execution state holds of the execution input.
1187///
1188/// The input is the `InputPayload` of the EXECUTION operation. That
1189/// operation may sit on a later page, so a page that lacks it differs from
1190/// a page whose EXECUTION operation has no payload. Only the first case
1191/// makes [`resolve_bootstrap_log`] keep searching the fetched pages.
1192#[derive(Debug, Clone, PartialEq, Eq)]
1193pub(crate) enum PageInput {
1194    /// The page holds no EXECUTION operation.
1195    NoExecutionOperation,
1196    /// The page holds the EXECUTION operation. The value is its
1197    /// `InputPayload`, or `None` when the payload is absent or JSON `null`.
1198    Execution(Option<String>),
1199}
1200
1201impl PageInput {
1202    /// Returns the `InputPayload`, or `None` when the page holds no
1203    /// EXECUTION operation or the operation has no payload.
1204    pub(crate) fn into_payload(self) -> Option<String> {
1205        match self {
1206            Self::NoExecutionOperation => None,
1207            Self::Execution(payload) => payload,
1208        }
1209    }
1210}
1211
1212/// Maps SDK `OperationStatus` to internal `CheckpointStatus`.
1213///
1214/// Every modeled variant maps explicitly; anything else, the generated
1215/// enum's unknown-variant carrier, or a variant added by an
1216/// aws-sdk-lambda upgrade before this list is revisited, maps to
1217/// [`CheckpointStatus::Unknown`] carrying the raw wire value, which replay
1218/// refuses to interpret (issue #45). The `operation_status_variant_canary`
1219/// test pins the modeled variant set, so a new upstream variant fails CI
1220/// for explicit classification instead of shipping through `Unknown`.
1221fn sdk_status_to_checkpoint_status(
1222    status: &aws_sdk_lambda::types::OperationStatus,
1223) -> CheckpointStatus {
1224    match status {
1225        aws_sdk_lambda::types::OperationStatus::Started => CheckpointStatus::Started,
1226        aws_sdk_lambda::types::OperationStatus::Pending => CheckpointStatus::Pending,
1227        aws_sdk_lambda::types::OperationStatus::Ready => CheckpointStatus::Ready,
1228        aws_sdk_lambda::types::OperationStatus::Succeeded => CheckpointStatus::Succeeded,
1229        aws_sdk_lambda::types::OperationStatus::Failed => CheckpointStatus::Failed,
1230        aws_sdk_lambda::types::OperationStatus::Cancelled => CheckpointStatus::Cancelled,
1231        aws_sdk_lambda::types::OperationStatus::TimedOut => CheckpointStatus::TimedOut,
1232        aws_sdk_lambda::types::OperationStatus::Stopped => CheckpointStatus::Stopped,
1233        other => CheckpointStatus::Unknown(other.as_str().to_owned()),
1234    }
1235}
1236
1237/// Extracts the result payload from an `Operation`.
1238///
1239/// `ChainedInvoke` results are deliberately excluded: they map to
1240/// `invoke_result` (see `operation_to_record`), matching the JSON path in
1241/// `parse_single_operation`.
1242fn extract_result(op: &Operation) -> Option<String> {
1243    op.step_details
1244        .as_ref()
1245        .and_then(|s| s.result.clone())
1246        .or_else(|| op.context_details.as_ref().and_then(|c| c.result.clone()))
1247        .or_else(|| {
1248            op.callback_details
1249                .as_ref()
1250                .and_then(|cb| cb.result.clone())
1251        })
1252}
1253
1254/// Extracts the error type from an `Operation`.
1255///
1256/// `ChainedInvoke` errors are deliberately excluded: they map to
1257/// `invoke_error_type` (see `operation_to_record`), matching the JSON path
1258/// in `parse_single_operation`.
1259fn extract_error_type(op: &Operation) -> Option<String> {
1260    op.step_details
1261        .as_ref()
1262        .and_then(|s| s.error.as_ref())
1263        .and_then(|e| e.error_type.clone())
1264        .or_else(|| {
1265            op.context_details
1266                .as_ref()
1267                .and_then(|c| c.error.as_ref())
1268                .and_then(|e| e.error_type.clone())
1269        })
1270        .or_else(|| {
1271            op.callback_details
1272                .as_ref()
1273                .and_then(|cb| cb.error.as_ref())
1274                .and_then(|e| e.error_type.clone())
1275        })
1276}
1277
1278/// Extracts the error message from an `Operation`.
1279///
1280/// `ChainedInvoke` errors are deliberately excluded: they map to
1281/// `invoke_error_message` (see `operation_to_record`), matching the JSON
1282/// path in `parse_single_operation`.
1283fn extract_error_message(op: &Operation) -> Option<String> {
1284    op.step_details
1285        .as_ref()
1286        .and_then(|s| s.error.as_ref())
1287        .and_then(|e| e.error_message.clone())
1288        .or_else(|| {
1289            op.context_details
1290                .as_ref()
1291                .and_then(|c| c.error.as_ref())
1292                .and_then(|e| e.error_message.clone())
1293        })
1294        .or_else(|| {
1295            op.callback_details
1296                .as_ref()
1297                .and_then(|cb| cb.error.as_ref())
1298                .and_then(|e| e.error_message.clone())
1299        })
1300}
1301
1302/// Extracts one field of the first error object present on the operation,
1303/// checking step, context, and callback details in that order (mirroring
1304/// [`extract_error_type`]).
1305fn extract_wire_error_field<T>(
1306    op: &Operation,
1307    read: impl Fn(&aws_sdk_lambda::types::ErrorObject) -> Option<T>,
1308) -> Option<T> {
1309    op.step_details
1310        .as_ref()
1311        .and_then(|s| s.error.as_ref())
1312        .and_then(&read)
1313        .or_else(|| {
1314            op.context_details
1315                .as_ref()
1316                .and_then(|c| c.error.as_ref())
1317                .and_then(&read)
1318        })
1319        .or_else(|| {
1320            op.callback_details
1321                .as_ref()
1322                .and_then(|cb| cb.error.as_ref())
1323                .and_then(&read)
1324        })
1325}
1326
1327#[cfg(test)]
1328mod tests {
1329    use super::*;
1330
1331    /// The `get_state` path reads `ParentId` and both timestamps.
1332    #[test]
1333    fn sdk_operation_parent_and_timestamps() {
1334        use std::time::{Duration, UNIX_EPOCH};
1335        let with_all = Operation::builder()
1336            .id("s")
1337            .r#type(OperationType::Step)
1338            .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
1339            .parent_id("p")
1340            .start_timestamp(aws_smithy_types::DateTime::from_secs(10))
1341            .end_timestamp(aws_smithy_types::DateTime::from_secs(12))
1342            .build()
1343            .expect("all required fields set");
1344        let (_, record) = operation_to_record(&with_all);
1345        assert_eq!(record.parent_id.as_deref(), Some("p"));
1346        assert_eq!(
1347            record.start_timestamp,
1348            Some(UNIX_EPOCH + Duration::from_secs(10))
1349        );
1350        assert_eq!(
1351            record.end_timestamp,
1352            Some(UNIX_EPOCH + Duration::from_secs(12))
1353        );
1354
1355        let bare = Operation::builder()
1356            .id("s")
1357            .r#type(OperationType::Step)
1358            .status(aws_sdk_lambda::types::OperationStatus::Started)
1359            .start_timestamp(aws_smithy_types::DateTime::from_secs(10))
1360            .build()
1361            .expect("all required fields set");
1362        let (_, record) = operation_to_record(&bare);
1363        assert_eq!(record.parent_id, None);
1364        assert_eq!(record.end_timestamp, None);
1365    }
1366
1367    /// The `get_state` path keeps the wire tokens of a type and a status
1368    /// this SDK version does not know.
1369    #[test]
1370    fn unknown_type_and_status_keep_wire_tokens_on_get_state_path() {
1371        let op = Operation::builder()
1372            .id("f")
1373            .r#type(OperationType::from("FUTURE_OP"))
1374            .status(aws_sdk_lambda::types::OperationStatus::from(
1375                "FUTURE_STATUS",
1376            ))
1377            .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
1378            .build()
1379            .expect("all required fields set");
1380        let (id, record) = operation_to_record(&op);
1381        let info = crate::plugin::OperationInfo::from_record(&id, &record);
1382        assert_eq!(info.kind(), None);
1383        assert_eq!(info.status(), None);
1384        assert_eq!(info.type_str(), Some("FUTURE_OP"));
1385        assert_eq!(info.status_str(), Some("FUTURE_STATUS"));
1386    }
1387
1388    /// Every known operation type is stored as its wire token.
1389    #[test]
1390    fn operation_type_to_string_returns_wire_tokens() {
1391        for (op_type, token) in [
1392            (OperationType::Callback, "CALLBACK"),
1393            (OperationType::ChainedInvoke, "CHAINED_INVOKE"),
1394            (OperationType::Context, "CONTEXT"),
1395            (OperationType::Execution, "EXECUTION"),
1396            (OperationType::Step, "STEP"),
1397            (OperationType::Wait, "WAIT"),
1398        ] {
1399            assert_eq!(operation_type_to_string(&op_type), token);
1400        }
1401    }
1402
1403    /// A fetched `EXECUTION` operation converts with its wire fields; any
1404    /// other type gives `None`.
1405    #[test]
1406    fn execution_info_from_operation_reads_execution_only() {
1407        use std::time::{Duration, UNIX_EPOCH};
1408        let execution = Operation::builder()
1409            .id("exec")
1410            .r#type(OperationType::Execution)
1411            .status(aws_sdk_lambda::types::OperationStatus::Started)
1412            .start_timestamp(aws_smithy_types::DateTime::from_secs(3))
1413            .end_timestamp(aws_smithy_types::DateTime::from_secs(4))
1414            .build()
1415            .expect("all required fields set");
1416        let info = execution_info_from_operation(&execution).expect("an EXECUTION operation");
1417        assert_eq!(info.id(), "exec");
1418        assert_eq!(info.type_str(), Some("EXECUTION"));
1419        assert_eq!(info.status_str(), Some("STARTED"));
1420        assert_eq!(info.parent_id(), None);
1421        assert_eq!(
1422            info.start_timestamp(),
1423            Some(UNIX_EPOCH + Duration::from_secs(3))
1424        );
1425        assert_eq!(
1426            info.end_timestamp(),
1427            Some(UNIX_EPOCH + Duration::from_secs(4))
1428        );
1429        let step = Operation::builder()
1430            .id("s")
1431            .r#type(OperationType::Step)
1432            .status(aws_sdk_lambda::types::OperationStatus::Started)
1433            .start_timestamp(aws_smithy_types::DateTime::from_secs(3))
1434            .build()
1435            .expect("all required fields set");
1436        assert!(execution_info_from_operation(&step).is_none());
1437    }
1438
1439    #[test]
1440    fn due_at_from_wait_and_step_details() {
1441        let op = |r#type: OperationType| {
1442            Operation::builder()
1443                .id("op")
1444                .r#type(r#type)
1445                .status(aws_sdk_lambda::types::OperationStatus::Started)
1446                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
1447        };
1448        let wait = op(OperationType::Wait)
1449            .wait_details(
1450                aws_sdk_lambda::types::WaitDetails::builder()
1451                    .scheduled_end_timestamp(aws_smithy_types::DateTime::from_secs(100))
1452                    .build(),
1453            )
1454            .build()
1455            .expect("all required fields set");
1456        let step = op(OperationType::Step)
1457            .step_details(
1458                aws_sdk_lambda::types::StepDetails::builder()
1459                    .next_attempt_timestamp(aws_smithy_types::DateTime::from_secs(200))
1460                    .build(),
1461            )
1462            .build()
1463            .expect("all required fields set");
1464        let neither = op(OperationType::Wait)
1465            .build()
1466            .expect("all required fields set");
1467        // Only a WAIT and a STEP carry an end time.
1468        let callback = op(OperationType::Callback)
1469            .wait_details(
1470                aws_sdk_lambda::types::WaitDetails::builder()
1471                    .scheduled_end_timestamp(aws_smithy_types::DateTime::from_secs(300))
1472                    .build(),
1473            )
1474            .build()
1475            .expect("all required fields set");
1476        let epoch = std::time::UNIX_EPOCH;
1477        assert_eq!(
1478            operation_to_record(&wait).1.due_at,
1479            Some(epoch + std::time::Duration::from_secs(100))
1480        );
1481        assert_eq!(
1482            operation_to_record(&step).1.due_at,
1483            Some(epoch + std::time::Duration::from_secs(200))
1484        );
1485        assert_eq!(operation_to_record(&neither).1.due_at, None);
1486        assert_eq!(operation_to_record(&callback).1.due_at, None);
1487    }
1488
1489    // Request mapping (aws-smithy-mocks)
1490    //
1491    // These tests mock the Lambda SDK client at the HTTP layer and pin the
1492    // per-operation request mapping of `LambdaExecutionClient`: what the
1493    // client puts on the wire, how it maps responses back, and how the
1494    // final error (after the SDK's own retry) maps into `ExecutionClientError`.
1495
1496    use aws_sdk_lambda::operation::checkpoint_durable_execution::{
1497        CheckpointDurableExecutionError, CheckpointDurableExecutionOutput,
1498    };
1499    use aws_sdk_lambda::operation::get_durable_execution_state::{
1500        GetDurableExecutionStateError, GetDurableExecutionStateOutput,
1501    };
1502    use aws_sdk_lambda::types::{CheckpointUpdatedExecutionState, OperationAction};
1503    use aws_smithy_mocks::{RuleMode, mock, mock_client};
1504
1505    /// Builds a minimal `OperationUpdate` for request-mapping assertions.
1506    fn make_update(id: &str, action: OperationAction) -> OperationUpdate {
1507        OperationUpdate::builder()
1508            .id(id)
1509            .r#type(OperationType::Step)
1510            .action(action)
1511            .build()
1512            .expect("valid OperationUpdate")
1513    }
1514
1515    /// `checkpoint` maps its arguments onto the `CheckpointDurableExecution`
1516    /// request, ARN, token, and every update, and maps the response's
1517    /// token, updated operations, and pagination marker back out.
1518    #[tokio::test]
1519    async fn checkpoint_maps_request_and_response() {
1520        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1521            .match_requests(|req| {
1522                req.durable_execution_arn() == Some("arn:aws:lambda:us-east-1:1:function:f")
1523                    && req.checkpoint_token() == Some("tok-in")
1524                    && req.updates().len() == 2
1525                    && req.updates().first().map(OperationUpdate::id) == Some("op-1")
1526                    && req.updates().get(1).map(OperationUpdate::id) == Some("op-2")
1527            })
1528            .then_output(|| {
1529                CheckpointDurableExecutionOutput::builder()
1530                    .checkpoint_token("tok-out")
1531                    .new_execution_state(
1532                        CheckpointUpdatedExecutionState::builder()
1533                            .operations(make_step_op("op-1", r#""r1""#))
1534                            .next_marker("more-pages")
1535                            .build(),
1536                    )
1537                    .build()
1538            });
1539        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1540        let client = LambdaExecutionClient::new(sdk_client);
1541
1542        let updates = vec![
1543            make_update("op-1", OperationAction::Start),
1544            make_update("op-2", OperationAction::Succeed),
1545        ];
1546        let output = client
1547            .checkpoint("arn:aws:lambda:us-east-1:1:function:f", "tok-in", updates)
1548            .await
1549            .expect("mocked checkpoint succeeds");
1550
1551        assert_eq!(output.checkpoint_token(), Some("tok-out"));
1552        assert_eq!(output.updated_operations.len(), 1);
1553        assert_eq!(
1554            output.updated_operations.first().map(Operation::id),
1555            Some("op-1")
1556        );
1557        assert_eq!(output.next_marker.as_deref(), Some("more-pages"));
1558        assert_eq!(rule.num_calls(), 1);
1559    }
1560
1561    /// The wire ID of the first root operation: the SHA-256 of `"1"`.
1562    const FIRST_ROOT_WIRE_ID: &str =
1563        "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b";
1564
1565    /// Drives `wrap` over `sdk_client` with a one-step handler on a fresh
1566    /// execution, so the step's first request is its START. Returns the
1567    /// response.
1568    ///
1569    /// The step awaits each checkpoint before it sends the next, so each
1570    /// checkpoint is its own request and a test can count the requests.
1571    async fn invoke_one_step(sdk_client: aws_sdk_lambda::Client) -> serde_json::Value {
1572        let service = crate::wrap(
1573            |_event: serde_json::Value, ctx: crate::DurableContext| async move {
1574                let value: u32 = ctx.step(|_| async { Ok(7_u32) }).name("one").await?;
1575                Ok::<u32, crate::BoxError>(value)
1576            },
1577            crate::Options::builder()
1578                .execution_client(Arc::new(LambdaExecutionClient::new(sdk_client)))
1579                .build()
1580                .expect("options build"),
1581        );
1582        let event = lambda_runtime::LambdaEvent::new(
1583            crate::InvocationPayload(serde_json::json!({
1584                "DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
1585                "CheckpointToken": "token-abc",
1586                "InitialExecutionState": {
1587                    "Operations": [
1588                        {
1589                            "Id": "root",
1590                            "Type": "Execution",
1591                            "Status": "STARTED",
1592                            "ExecutionDetails": { "InputPayload": "{}" }
1593                        }
1594                    ]
1595                }
1596            })),
1597            lambda_runtime::Context::default(),
1598        );
1599        service(event)
1600            .await
1601            .map(|response| response.0)
1602            .expect("a tokenless response answers PENDING, not an invocation error")
1603    }
1604
1605    /// Every mocked checkpoint response has no `CheckpointToken`.
1606    ///
1607    /// (a) The transport reports the response as `Ok` with no token.
1608    /// (b) A request without an `EXECUTION` update withdraws the token: the
1609    /// handler answers exactly `{"Status":"PENDING"}` after one request,
1610    /// the step's START, and sends no terminal `FAIL`.
1611    /// (c) A request with an `EXECUTION` update is exempt: the call returns
1612    /// `Ok`, no latch is set, the phase stays `Running`, and no WARN is
1613    /// logged.
1614    #[tokio::test]
1615    async fn checkpoint_output_without_token_answers_pending_unless_execution_update() {
1616        use std::sync::{Arc, Mutex as StdMutex};
1617
1618        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1619            .then_output(|| CheckpointDurableExecutionOutput::builder().build());
1620        let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
1621        let output = client
1622            .checkpoint("arn:test", "tok", Vec::new())
1623            .await
1624            .expect("a tokenless response is Ok");
1625        assert_eq!(output.checkpoint_token(), None);
1626
1627        // Capture each request's updates.
1628        let requests: Arc<StdMutex<Vec<Vec<OperationUpdate>>>> = Arc::default();
1629        let seen = Arc::clone(&requests);
1630        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1631            .match_requests(move |req| {
1632                seen.lock()
1633                    .unwrap_or_else(std::sync::PoisonError::into_inner)
1634                    .push(req.updates().to_vec());
1635                true
1636            })
1637            .then_output(|| CheckpointDurableExecutionOutput::builder().build());
1638        let response =
1639            invoke_one_step(mock_client!(aws_sdk_lambda, RuleMode::MatchAny, [&rule])).await;
1640        assert_eq!(response, serde_json::json!({ "Status": "PENDING" }));
1641        let requests = requests
1642            .lock()
1643            .unwrap_or_else(std::sync::PoisonError::into_inner)
1644            .clone();
1645        assert_eq!(requests.len(), 1, "{requests:?}");
1646        let first = requests.first().expect("one request");
1647        assert_eq!(first.len(), 1, "{first:?}");
1648        let start = first.first().expect("the step's START");
1649        assert_eq!(start.id(), FIRST_ROOT_WIRE_ID);
1650        assert_eq!(start.action(), &OperationAction::Start);
1651        assert!(
1652            requests
1653                .iter()
1654                .flatten()
1655                .all(|u| u.action() != &OperationAction::Fail),
1656            "no terminal FAIL is sent: {requests:?}"
1657        );
1658
1659        let (messages, _guard) = crate::driver::test_support::EventMessages::install();
1660        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1661            .then_output(|| CheckpointDurableExecutionOutput::builder().build());
1662        let ctx = crate::DurableContext::new_root_with_client(
1663            "arn:test".to_owned(),
1664            lambda_runtime::Context::default(),
1665            Arc::new(CheckpointLog::empty()),
1666            Arc::new(LambdaExecutionClient::new(mock_client!(
1667                aws_sdk_lambda,
1668                [&rule]
1669            ))),
1670            "tok".to_owned(),
1671        );
1672        let succeed = OperationUpdate::builder()
1673            .id("root")
1674            .r#type(OperationType::Execution)
1675            .action(OperationAction::Succeed)
1676            .payload("1")
1677            .build()
1678            .expect("valid update");
1679        ctx.checkpoint_updates_direct(vec![succeed])
1680            .await
1681            .expect("an EXECUTION update is exempt from withdrawal");
1682        assert!(!ctx.coordinator().withdrawn());
1683        assert_eq!(
1684            ctx.coordinator().phase(),
1685            crate::coordinator::Phase::Running
1686        );
1687        assert!(
1688            messages.at(tracing::Level::WARN).is_empty(),
1689            "{:?}",
1690            messages.at(tracing::Level::WARN)
1691        );
1692    }
1693
1694    /// `ExecutionClientError::new` keeps the scope and the message, has no
1695    /// source, and `is_retryable` derives from the scope.
1696    #[test]
1697    fn execution_client_error_new_keeps_scope_and_message() {
1698        use std::error::Error as _;
1699
1700        let err = ExecutionClientError::new(ExecutionClientErrorScope::Execution, "x");
1701        assert_eq!(err.scope(), ExecutionClientErrorScope::Execution);
1702        assert!(!err.is_retryable());
1703        assert!(err.source().is_none());
1704        assert_eq!(err.to_string(), "durable: client: x");
1705
1706        let err = ExecutionClientError::new(ExecutionClientErrorScope::Invocation, "x");
1707        assert_eq!(err.scope(), ExecutionClientErrorScope::Invocation);
1708        assert!(err.is_retryable());
1709    }
1710
1711    /// The Lambda transport keeps the final `SdkError` as the source, and
1712    /// keeps today's `DisplayErrorContext` text as the message.
1713    #[tokio::test]
1714    async fn lambda_transport_error_source_downcasts_to_sdk_error() {
1715        use std::error::Error as _;
1716
1717        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1718            .sequence()
1719            .http_status(403, None)
1720            .build();
1721        let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
1722        let err = client
1723            .checkpoint("arn:test", "tok", Vec::new())
1724            .await
1725            .expect_err("a 403 fails the call");
1726
1727        let source = err.source().expect("the transport keeps the SdkError");
1728        let sdk = source
1729            .downcast_ref::<CheckpointSdkError>()
1730            .expect("the source is the final SdkError");
1731        let text = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(sdk));
1732        assert_eq!(err.message(), text);
1733        assert_eq!(format!("{err:#}"), format!("durable: client: {text}"));
1734        assert_eq!(err.scope(), ExecutionClientErrorScope::Execution);
1735    }
1736
1737    /// A checkpoint call whose final outcome is a modeled parameter
1738    /// rejection, a deterministic client fault the SDK does not retry,
1739    /// classifies NON-retryable: re-invoking replays into the same
1740    /// rejection, so the recovery is failing the execution, not another
1741    /// lap.
1742    #[tokio::test]
1743    async fn checkpoint_final_error_maps_to_non_retryable_client_error() {
1744        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_error(|| {
1745            CheckpointDurableExecutionError::InvalidParameterValueException(
1746                aws_sdk_lambda::types::error::InvalidParameterValueException::builder().build(),
1747            )
1748        });
1749        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1750        let client = LambdaExecutionClient::new(sdk_client);
1751
1752        let err = client
1753            .checkpoint("arn:test", "tok", Vec::new())
1754            .await
1755            .expect_err("modeled error must fail the call");
1756        assert!(!err.is_retryable());
1757        assert_eq!(rule.num_calls(), 1, "a client fault is not retried");
1758    }
1759
1760    /// The backend reports stale checkpoint tokens as an
1761    /// `InvalidParameterValueException` whose message starts with
1762    /// `Invalid checkpoint token`; the production client must classify
1763    /// that modeled 4xx as retryable so the invocation fails and durable
1764    /// execution can resume with a fresh token.
1765    #[tokio::test]
1766    async fn checkpoint_stale_token_error_maps_to_retryable_client_error() {
1767        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_error(|| {
1768            CheckpointDurableExecutionError::InvalidParameterValueException(
1769                aws_sdk_lambda::types::error::InvalidParameterValueException::builder()
1770                    .message("Invalid checkpoint token: token has been superseded")
1771                    .build(),
1772            )
1773        });
1774        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1775        let client = LambdaExecutionClient::new(sdk_client);
1776
1777        let err = client
1778            .checkpoint("arn:test", "tok", Vec::new())
1779            .await
1780            .expect_err("stale checkpoint token must fail the call");
1781        assert!(err.is_retryable());
1782        assert_eq!(rule.num_calls(), 1, "a client fault is not retried");
1783    }
1784
1785    /// Transient-failure retry belongs to the aws-sdk's standard retry and
1786    /// nothing else: a persistent 503 is attempted exactly the SDK's
1787    /// default 3 times, not 9, which the deleted hand-rolled outer loop
1788    /// used to multiply it to, and the exhausted call maps to a retryable
1789    /// `ExecutionClientError`.
1790    #[tokio::test]
1791    async fn checkpoint_retry_is_the_sdk_standard_retry_alone() {
1792        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
1793            .sequence()
1794            .http_status(503, None)
1795            .repeatedly()
1796            .build();
1797        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1798        let client = LambdaExecutionClient::new(sdk_client);
1799
1800        let err = client
1801            .checkpoint("arn:test", "tok", Vec::new())
1802            .await
1803            .expect_err("persistent 503 must exhaust retries");
1804        assert!(err.is_retryable());
1805        assert_eq!(
1806            rule.num_calls(),
1807            3,
1808            "attempts must equal the SDK standard-retry budget — no outer loop multiplying them"
1809        );
1810    }
1811
1812    /// `get_state` sends one request with the given marker and `MaxItems`
1813    /// 1000, and returns that page and its `NextMarker`.
1814    #[tokio::test]
1815    async fn get_state_returns_one_page_with_marker() {
1816        let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
1817            .match_requests(|req| {
1818                req.durable_execution_arn() == Some("arn:test")
1819                    && req.checkpoint_token() == Some("tok")
1820                    && req.marker() == Some("m-1")
1821                    && req.max_items() == Some(1000)
1822            })
1823            .then_output(|| {
1824                GetDurableExecutionStateOutput::builder()
1825                    .operations(make_step_op("op-1", r#""p1""#))
1826                    .operations(make_step_op("op-2", r#""p2""#))
1827                    .next_marker("m-2")
1828                    .build()
1829                    .expect("operations is set")
1830            });
1831        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
1832        let client = LambdaExecutionClient::new(sdk_client);
1833
1834        let page = client
1835            .get_state("arn:test", "tok", Some("m-1"))
1836            .await
1837            .expect("get_state succeeds");
1838
1839        assert_eq!(rule.num_calls(), 1);
1840        assert_eq!(page.operations.len(), 2);
1841        assert_eq!(page.operations.first().map(Operation::id), Some("op-1"));
1842        assert_eq!(page.operations.get(1).map(Operation::id), Some("op-2"));
1843        assert_eq!(page.next_marker(), Some("m-2"));
1844    }
1845
1846    /// A transport that returns three pages, the last without a marker,
1847    /// and records the marker of every call.
1848    struct ThreePageClient {
1849        markers: Mutex<Vec<Option<String>>>,
1850    }
1851
1852    impl ExecutionClient for ThreePageClient {
1853        fn checkpoint(
1854            &self,
1855            _execution_arn: &str,
1856            _checkpoint_token: &str,
1857            _updates: Vec<OperationUpdate>,
1858        ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
1859        {
1860            Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
1861        }
1862
1863        fn get_state(
1864            &self,
1865            _execution_arn: &str,
1866            _checkpoint_token: &str,
1867            marker: Option<&str>,
1868        ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
1869        {
1870            self.markers
1871                .lock()
1872                .unwrap_or_else(std::sync::PoisonError::into_inner)
1873                .push(marker.map(str::to_owned));
1874            let page = match marker {
1875                None => GetStateOutput::new(vec![make_step_op("a", "1")], Some("p-2".to_owned())),
1876                Some("p-2") => {
1877                    GetStateOutput::new(vec![make_step_op("b", "2")], Some("p-3".to_owned()))
1878                }
1879                Some(_) => GetStateOutput::new(vec![make_step_op("c", "3")], None),
1880            };
1881            Box::pin(async move { Ok(page) })
1882        }
1883    }
1884
1885    /// The whole-state helper follows each marker until the last page and
1886    /// concatenates the pages in order.
1887    #[tokio::test]
1888    async fn sdk_loops_get_state_until_marker_empty() {
1889        let client = ThreePageClient {
1890            markers: Mutex::new(Vec::new()),
1891        };
1892        let operations = get_full_state(&client, "arn:test", "tok")
1893            .await
1894            .expect("whole state");
1895
1896        let ids: Vec<&str> = operations.iter().map(Operation::id).collect();
1897        assert_eq!(ids, ["a", "b", "c"]);
1898        let markers = client
1899            .markers
1900            .lock()
1901            .unwrap_or_else(std::sync::PoisonError::into_inner)
1902            .clone();
1903        assert_eq!(
1904            markers,
1905            [None, Some("p-2".to_owned()), Some("p-3".to_owned())]
1906        );
1907    }
1908
1909    /// A transport that serves page 2 for marker `m-1` and page 3 for
1910    /// marker `m-2`, and records the marker of every call. A call without
1911    /// a marker, a re-read of the first page, gets a step `s0` that the
1912    /// test then finds in the log.
1913    struct InlineContinuationClient {
1914        markers: Mutex<Vec<Option<String>>>,
1915    }
1916
1917    impl ExecutionClient for InlineContinuationClient {
1918        fn checkpoint(
1919            &self,
1920            _execution_arn: &str,
1921            _checkpoint_token: &str,
1922            _updates: Vec<OperationUpdate>,
1923        ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
1924        {
1925            Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
1926        }
1927
1928        fn get_state(
1929            &self,
1930            _execution_arn: &str,
1931            _checkpoint_token: &str,
1932            marker: Option<&str>,
1933        ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
1934        {
1935            self.markers
1936                .lock()
1937                .unwrap_or_else(std::sync::PoisonError::into_inner)
1938                .push(marker.map(str::to_owned));
1939            let page = match marker {
1940                Some("m-1") => {
1941                    GetStateOutput::new(vec![make_step_op("s2", "2")], Some("m-2".to_owned()))
1942                }
1943                Some("m-2") => GetStateOutput::new(vec![make_step_op("s3", "3")], None),
1944                _ => GetStateOutput::new(vec![make_step_op("s0", "0")], None),
1945            };
1946            Box::pin(async move { Ok(page) })
1947        }
1948    }
1949
1950    /// Bootstrap keeps the inline page and continues from its marker: it
1951    /// never re-reads the first page, and the merged log holds the inline
1952    /// step and the steps of both fetched pages.
1953    #[tokio::test]
1954    async fn bootstrap_keeps_inline_page_and_continues_from_marker() {
1955        let client = InlineContinuationClient {
1956            markers: Mutex::new(Vec::new()),
1957        };
1958        let execution = Operation::builder()
1959            .id("execution")
1960            .r#type(OperationType::Execution)
1961            .status(aws_sdk_lambda::types::OperationStatus::Started)
1962            .start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
1963            .execution_details(
1964                aws_sdk_lambda::types::ExecutionDetails::builder()
1965                    .input_payload("{}")
1966                    .build(),
1967            )
1968            .build()
1969            .expect("id, type, status and start timestamp are set");
1970        let execution_info = execution_info_from_operation(&execution);
1971        let first_page = BootstrapState {
1972            log: operations_to_checkpoint_log(&[execution, make_step_op("s1", "1")]),
1973            input_payload: Some("{}".to_owned()),
1974            execution: execution_info,
1975        };
1976
1977        let BootstrapState {
1978            log,
1979            input_payload: input,
1980            ..
1981        } = resolve_bootstrap_log(&client, "arn:test", "tok", first_page, Some("m-1"))
1982            .await
1983            .expect("bootstrap succeeds");
1984
1985        let markers = client
1986            .markers
1987            .lock()
1988            .unwrap_or_else(std::sync::PoisonError::into_inner)
1989            .clone();
1990        assert_eq!(markers, [Some("m-1".to_owned()), Some("m-2".to_owned())]);
1991        for id in ["s1", "s2", "s3"] {
1992            assert!(log.get(id).is_some(), "{id} must be in the merged log");
1993        }
1994        assert!(
1995            log.get("s0").is_none(),
1996            "the first page must not be re-read"
1997        );
1998        assert_eq!(input.as_deref(), Some("{}"));
1999    }
2000
2001    /// Builds an EXECUTION operation with the given `InputPayload`, or with
2002    /// none when `input` is `None`.
2003    fn make_execution_op(id: &str, input: Option<&str>) -> Operation {
2004        let mut details = aws_sdk_lambda::types::ExecutionDetails::builder();
2005        if let Some(input) = input {
2006            details = details.input_payload(input);
2007        }
2008        Operation::builder()
2009            .id(id)
2010            .r#type(OperationType::Execution)
2011            .status(aws_sdk_lambda::types::OperationStatus::Started)
2012            .start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
2013            .execution_details(details.build())
2014            .build()
2015            .expect("id, type, status and start timestamp are set")
2016    }
2017
2018    /// A transport that serves a fixed page for each marker and records the
2019    /// marker of every call. A marker it does not know gets an empty last
2020    /// page.
2021    struct MarkerPagesClient {
2022        pages: Vec<(&'static str, Vec<Operation>, Option<&'static str>)>,
2023        markers: Mutex<Vec<Option<String>>>,
2024    }
2025
2026    impl ExecutionClient for MarkerPagesClient {
2027        fn checkpoint(
2028            &self,
2029            _execution_arn: &str,
2030            _checkpoint_token: &str,
2031            _updates: Vec<OperationUpdate>,
2032        ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
2033        {
2034            Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
2035        }
2036
2037        fn get_state(
2038            &self,
2039            _execution_arn: &str,
2040            _checkpoint_token: &str,
2041            marker: Option<&str>,
2042        ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
2043        {
2044            self.markers
2045                .lock()
2046                .unwrap_or_else(std::sync::PoisonError::into_inner)
2047                .push(marker.map(str::to_owned));
2048            let page = self
2049                .pages
2050                .iter()
2051                .find(|(key, _, _)| Some(*key) == marker)
2052                .map_or_else(
2053                    || GetStateOutput::new(vec![], None),
2054                    |(_, ops, next)| GetStateOutput::new(ops.clone(), next.map(str::to_owned)),
2055                );
2056            Box::pin(async move { Ok(page) })
2057        }
2058    }
2059
2060    /// Resolves a bootstrap whose inline page holds no operation and has
2061    /// marker `m-1`, against the given fetched pages. Returns the input and
2062    /// the recorded markers.
2063    async fn resolve_input_from_pages(
2064        pages: Vec<(&'static str, Vec<Operation>, Option<&'static str>)>,
2065    ) -> (Option<String>, Vec<Option<String>>) {
2066        let client = MarkerPagesClient {
2067            pages,
2068            markers: Mutex::new(Vec::new()),
2069        };
2070        let BootstrapState {
2071            input_payload: input,
2072            ..
2073        } = resolve_bootstrap_log(
2074            &client,
2075            "arn:test",
2076            "tok",
2077            BootstrapState {
2078                log: operations_to_checkpoint_log(&[]),
2079                input_payload: None,
2080                execution: None,
2081            },
2082            Some("m-1"),
2083        )
2084        .await
2085        .expect("bootstrap succeeds");
2086        let markers = client
2087            .markers
2088            .lock()
2089            .unwrap_or_else(std::sync::PoisonError::into_inner)
2090            .clone();
2091        (input, markers)
2092    }
2093
2094    /// When the inline page holds no EXECUTION operation, the input comes
2095    /// from the first EXECUTION operation of the fetched pages, in page
2096    /// order. The search continues past a fetched page that holds none.
2097    /// The first EXECUTION operation decides the input even when its
2098    /// `InputPayload` is absent, so a later page's payload does not replace
2099    /// it.
2100    #[tokio::test]
2101    async fn bootstrap_input_comes_from_first_fetched_execution_operation() {
2102        let (input, markers) = resolve_input_from_pages(vec![
2103            ("m-1", vec![make_step_op("s1", "1")], Some("m-2")),
2104            (
2105                "m-2",
2106                vec![make_execution_op("execution", Some(r#"{"count":1}"#))],
2107                Some("m-3"),
2108            ),
2109            (
2110                "m-3",
2111                vec![make_execution_op("late", Some(r#"{"late":1}"#))],
2112                None,
2113            ),
2114        ])
2115        .await;
2116        assert_eq!(input.as_deref(), Some(r#"{"count":1}"#));
2117        assert_eq!(
2118            markers,
2119            [
2120                Some("m-1".to_owned()),
2121                Some("m-2".to_owned()),
2122                Some("m-3".to_owned())
2123            ]
2124        );
2125
2126        let (input, markers) = resolve_input_from_pages(vec![
2127            ("m-1", vec![make_step_op("s1", "1")], Some("m-2")),
2128            (
2129                "m-2",
2130                vec![make_execution_op("execution", None)],
2131                Some("m-3"),
2132            ),
2133            (
2134                "m-3",
2135                vec![make_execution_op("late", Some(r#"{"late":1}"#))],
2136                None,
2137            ),
2138        ])
2139        .await;
2140        assert_eq!(input, None, "an absent first payload must not be replaced");
2141        assert_eq!(markers.len(), 3, "pagination still reaches the last page");
2142    }
2143
2144    /// The accessors of a transport's output read an empty token or an
2145    /// empty marker as `None`, so a custom transport that passes the
2146    /// service's empty string through behaves like an absent value.
2147    #[test]
2148    fn output_accessors_read_empty_values_as_none() {
2149        let output = CheckpointOutput::new(Some(String::new()), vec![], None);
2150        assert_eq!(output.checkpoint_token(), None);
2151        let page = GetStateOutput::new(vec![], Some(String::new()));
2152        assert_eq!(page.next_marker(), None);
2153    }
2154
2155    /// A transport whose first page carries an empty marker.
2156    struct EmptyMarkerClient {
2157        calls: Mutex<u32>,
2158    }
2159
2160    impl ExecutionClient for EmptyMarkerClient {
2161        fn checkpoint(
2162            &self,
2163            _execution_arn: &str,
2164            _checkpoint_token: &str,
2165            _updates: Vec<OperationUpdate>,
2166        ) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
2167        {
2168            Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
2169        }
2170
2171        fn get_state(
2172            &self,
2173            _execution_arn: &str,
2174            _checkpoint_token: &str,
2175            _marker: Option<&str>,
2176        ) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
2177        {
2178            *self
2179                .calls
2180                .lock()
2181                .unwrap_or_else(std::sync::PoisonError::into_inner) += 1;
2182            let page = GetStateOutput::new(vec![make_step_op("a", "1")], Some(String::new()));
2183            Box::pin(async move { Ok(page) })
2184        }
2185    }
2186
2187    /// The whole-state helper treats an empty marker as the last page: it
2188    /// makes one call and returns that page.
2189    #[tokio::test]
2190    async fn sdk_stops_paginating_on_empty_marker() {
2191        let client = EmptyMarkerClient {
2192            calls: Mutex::new(0),
2193        };
2194        let operations = get_full_state(&client, "arn:test", "tok")
2195            .await
2196            .expect("whole state");
2197
2198        let ids: Vec<&str> = operations.iter().map(Operation::id).collect();
2199        assert_eq!(ids, ["a"]);
2200        assert_eq!(
2201            *client
2202                .calls
2203                .lock()
2204                .unwrap_or_else(std::sync::PoisonError::into_inner),
2205            1
2206        );
2207    }
2208
2209    /// `checkpoint` stores an empty `CheckpointToken` and an empty
2210    /// `NextMarker` as `None`. `DurableContext` reads the marker field
2211    /// directly to decide whether to paginate, so the transport itself must
2212    /// normalize it.
2213    #[tokio::test]
2214    async fn checkpoint_maps_empty_token_and_marker_to_none() {
2215        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_output(|| {
2216            CheckpointDurableExecutionOutput::builder()
2217                .checkpoint_token("")
2218                .new_execution_state(
2219                    CheckpointUpdatedExecutionState::builder()
2220                        .operations(make_step_op("op-1", r#""r1""#))
2221                        .next_marker("")
2222                        .build(),
2223                )
2224                .build()
2225        });
2226        let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
2227
2228        let output = client
2229            .checkpoint("arn:test", "tok", Vec::new())
2230            .await
2231            .expect("mocked checkpoint succeeds");
2232
2233        assert_eq!(output.checkpoint_token, None);
2234        assert_eq!(output.next_marker, None);
2235        assert_eq!(output.updated_operations.len(), 1);
2236    }
2237
2238    /// `get_state` stores an empty `NextMarker` as `None`.
2239    #[tokio::test]
2240    async fn get_state_maps_empty_marker_to_none() {
2241        let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_output(|| {
2242            GetDurableExecutionStateOutput::builder()
2243                .operations(make_step_op("op-1", r#""p1""#))
2244                .next_marker("")
2245                .build()
2246                .expect("operations is set")
2247        });
2248        let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
2249
2250        let page = client
2251            .get_state("arn:test", "tok", None)
2252            .await
2253            .expect("get_state succeeds");
2254
2255        assert_eq!(page.next_marker, None);
2256        assert_eq!(page.operations.len(), 1);
2257    }
2258
2259    /// A `get_state` failure maps into a retryable `ExecutionClientError` exactly
2260    /// like a checkpoint failure.
2261    #[tokio::test]
2262    async fn get_state_final_error_maps_to_retryable_client_error() {
2263        let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_error(|| {
2264            GetDurableExecutionStateError::InvalidParameterValueException(
2265                aws_sdk_lambda::types::error::InvalidParameterValueException::builder().build(),
2266            )
2267        });
2268        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2269        let client = LambdaExecutionClient::new(sdk_client);
2270
2271        let err = client
2272            .get_state("arn:test", "tok", None)
2273            .await
2274            .expect_err("modeled error must fail the call");
2275        assert!(err.is_retryable());
2276    }
2277
2278    /// An unmodeled 403 on a checkpoint, such as an `AccessDeniedException`
2279    /// when the role lacks `lambda:CheckpointDurableExecution`, fails the
2280    /// execution. Standard retry does not retry it.
2281    #[tokio::test]
2282    async fn checkpoint_unmodeled_403_fails_the_execution() {
2283        let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
2284            .sequence()
2285            .http_status(403, None)
2286            .repeatedly()
2287            .build();
2288        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2289        let client = LambdaExecutionClient::new(sdk_client);
2290
2291        let err = client
2292            .checkpoint("arn:test", "tok", Vec::new())
2293            .await
2294            .expect_err("a 403 must fail the call");
2295        assert!(!err.is_retryable());
2296        assert_eq!(rule.num_calls(), 1, "a 403 is not retried");
2297    }
2298
2299    /// The four KMS errors of `GetDurableExecutionState` are a function
2300    /// misconfiguration that a re-invocation cannot fix, so each one is
2301    /// non-retryable, and the message names the exception.
2302    #[tokio::test]
2303    async fn get_state_kms_errors_classify_non_retryable() {
2304        use aws_sdk_lambda::types::error::{
2305            KmsAccessDeniedException, KmsDisabledException, KmsInvalidStateException,
2306            KmsNotFoundException,
2307        };
2308        type Make = fn() -> GetDurableExecutionStateError;
2309        let cases: [(Make, &str); 4] = [
2310            (
2311                || {
2312                    GetDurableExecutionStateError::KmsAccessDeniedException(
2313                        KmsAccessDeniedException::builder().build(),
2314                    )
2315                },
2316                "KMSAccessDeniedException",
2317            ),
2318            (
2319                || {
2320                    GetDurableExecutionStateError::KmsDisabledException(
2321                        KmsDisabledException::builder().build(),
2322                    )
2323                },
2324                "KMSDisabledException",
2325            ),
2326            (
2327                || {
2328                    GetDurableExecutionStateError::KmsInvalidStateException(
2329                        KmsInvalidStateException::builder().build(),
2330                    )
2331                },
2332                "KMSInvalidStateException",
2333            ),
2334            (
2335                || {
2336                    GetDurableExecutionStateError::KmsNotFoundException(
2337                        KmsNotFoundException::builder().build(),
2338                    )
2339                },
2340                "KMSNotFoundException",
2341            ),
2342        ];
2343        for (make, wire_name) in cases {
2344            let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_error(make);
2345            let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2346            let client = LambdaExecutionClient::new(sdk_client);
2347
2348            let err = client
2349                .get_state("arn:test", "tok", None)
2350                .await
2351                .expect_err("a KMS error must fail the call");
2352            assert!(!err.is_retryable(), "{wire_name} must be non-retryable");
2353            assert!(
2354                err.to_string().contains(wire_name),
2355                "message must name {wire_name}, got: {err}"
2356            );
2357        }
2358    }
2359
2360    /// A plain 4xx on the state read stays retryable, as in JS, which
2361    /// rethrows it and so fails only the invocation.
2362    #[tokio::test]
2363    async fn get_state_plain_4xx_stays_retryable() {
2364        let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
2365            .sequence()
2366            .http_status(403, None)
2367            .repeatedly()
2368            .build();
2369        let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
2370        let client = LambdaExecutionClient::new(sdk_client);
2371
2372        let err = client
2373            .get_state("arn:test", "tok", None)
2374            .await
2375            .expect_err("a 403 must fail the call");
2376        assert!(err.is_retryable());
2377    }
2378
2379    // Test double behavior
2380
2381    /// A queued `AcceptThenRetryableError` records the request's updates,
2382    /// then answers with a retryable error carrying its message.
2383    #[tokio::test]
2384    async fn accept_then_retryable_error_records_then_fails() {
2385        let client = InMemoryExecutionClient::new(Vec::new());
2386        client.enqueue_checkpoint_response(TestResponse::AcceptThenRetryableError(
2387            "lost response".to_owned(),
2388        ));
2389        let update = OperationUpdate::builder()
2390            .id("e")
2391            .r#type(OperationType::Execution)
2392            .action(OperationAction::Succeed)
2393            .build()
2394            .expect("valid update");
2395        let err = client
2396            .checkpoint("arn", "token", vec![update])
2397            .await
2398            .expect_err("a retryable error");
2399        assert!(err.is_retryable());
2400        assert_eq!(err.message(), "lost response");
2401        assert_eq!(client.recorded_updates().len(), 1);
2402    }
2403
2404    #[tokio::test]
2405    async fn in_memory_client_returns_queued_responses_in_order() {
2406        let client = InMemoryExecutionClient::new(Vec::new());
2407        client.enqueue_checkpoint_response(TestResponse::RetryableError("throttled".to_owned()));
2408        client.enqueue_checkpoint_response(TestResponse::NonRetryableError("invalid".to_owned()));
2409        client.enqueue_checkpoint_response(TestResponse::Success(Vec::new()));
2410
2411        let first = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2412        let first_err = first.unwrap_err();
2413        assert!(first_err.is_retryable());
2414
2415        let second = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2416        let second_err = second.unwrap_err();
2417        assert!(!second_err.is_retryable());
2418
2419        let third = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2420        assert!(third.is_ok());
2421
2422        let count = *client
2423            .checkpoint_call_count
2424            .lock()
2425            .unwrap_or_else(std::sync::PoisonError::into_inner);
2426        assert_eq!(count, 3);
2427    }
2428
2429    #[tokio::test]
2430    async fn success_returns_checkpoint_output() {
2431        let client = InMemoryExecutionClient::new(Vec::new());
2432        let result = client.checkpoint("arn:test", "token-0", Vec::new()).await;
2433
2434        assert!(result.is_ok());
2435        let output = result.unwrap();
2436        assert!(
2437            output
2438                .checkpoint_token()
2439                .is_some_and(|t| t.starts_with("token-"))
2440        );
2441    }
2442
2443    // Checkpoint log population
2444
2445    #[test]
2446    fn checkpoint_log_from_operations_maps_statuses() {
2447        let ops = vec![
2448            Operation::builder()
2449                .id("op-1")
2450                .r#type(OperationType::Step)
2451                .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2452                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2453                .step_details(
2454                    aws_sdk_lambda::types::StepDetails::builder()
2455                        .attempt(1)
2456                        .result(r#""hello""#)
2457                        .build(),
2458                )
2459                .build()
2460                .unwrap(),
2461            Operation::builder()
2462                .id("op-2")
2463                .r#type(OperationType::Step)
2464                .status(aws_sdk_lambda::types::OperationStatus::Failed)
2465                .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2466                .step_details(
2467                    aws_sdk_lambda::types::StepDetails::builder()
2468                        .attempt(2)
2469                        .error(
2470                            aws_sdk_lambda::types::ErrorObject::builder()
2471                                .error_type("StepError")
2472                                .error_message("oops")
2473                                .build(),
2474                        )
2475                        .build(),
2476                )
2477                .build()
2478                .unwrap(),
2479        ];
2480
2481        let log = operations_to_checkpoint_log(&ops);
2482
2483        let r1 = log.get("op-1");
2484        assert!(r1.is_some());
2485        let r1 = r1.unwrap();
2486        assert_eq!(r1.status, CheckpointStatus::Succeeded);
2487        assert_eq!(r1.result.as_deref(), Some(r#""hello""#));
2488        assert_eq!(r1.attempt, 1);
2489
2490        let r2 = log.get("op-2");
2491        assert!(r2.is_some());
2492        let r2 = r2.unwrap();
2493        assert_eq!(r2.status, CheckpointStatus::Failed);
2494        assert_eq!(r2.error_type.as_deref(), Some("StepError"));
2495        assert_eq!(r2.error_message.as_deref(), Some("oops"));
2496        assert_eq!(r2.attempt, 2);
2497    }
2498
2499    /// A paginated `ChainedInvoke` operation converts into the
2500    /// invoke-specific record fields (`invoke_result` /
2501    /// `invoke_error_*`) that invoke replay reads, matching the inline
2502    /// JSON conversion in `parse_single_operation`, and never leaks into
2503    /// the generic `result` / `error_*` fields.
2504    #[test]
2505    fn chained_invoke_maps_to_invoke_specific_fields() {
2506        let ops = vec![
2507            Operation::builder()
2508                .id("invoke-ok")
2509                .r#type(OperationType::ChainedInvoke)
2510                .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2511                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2512                .chained_invoke_details(
2513                    aws_sdk_lambda::types::ChainedInvokeDetails::builder()
2514                        .result(r#""invoke-payload""#)
2515                        .build(),
2516                )
2517                .build()
2518                .unwrap(),
2519            Operation::builder()
2520                .id("invoke-err")
2521                .r#type(OperationType::ChainedInvoke)
2522                .status(aws_sdk_lambda::types::OperationStatus::Failed)
2523                .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2524                .chained_invoke_details(
2525                    aws_sdk_lambda::types::ChainedInvokeDetails::builder()
2526                        .error(
2527                            aws_sdk_lambda::types::ErrorObject::builder()
2528                                .error_type("InvokeError")
2529                                .error_message("downstream failed")
2530                                .build(),
2531                        )
2532                        .build(),
2533                )
2534                .build()
2535                .unwrap(),
2536        ];
2537
2538        let log = operations_to_checkpoint_log(&ops);
2539
2540        let ok = log.get("invoke-ok");
2541        assert!(ok.is_some());
2542        let ok = ok.unwrap();
2543        assert_eq!(ok.invoke_result.as_deref(), Some(r#""invoke-payload""#));
2544        assert_eq!(
2545            ok.result, None,
2546            "invoke results must not leak into the generic result field"
2547        );
2548
2549        let err = log.get("invoke-err");
2550        assert!(err.is_some());
2551        let err = err.unwrap();
2552        assert_eq!(err.invoke_error_type.as_deref(), Some("InvokeError"));
2553        assert_eq!(
2554            err.invoke_error_message.as_deref(),
2555            Some("downstream failed")
2556        );
2557        assert_eq!(
2558            err.error_type, None,
2559            "invoke errors must not leak into the generic error fields"
2560        );
2561        assert_eq!(err.error_message, None);
2562    }
2563
2564    /// A paginated child-context operation preserves
2565    /// `ContextDetails.ReplayChildren`, which large child contexts require
2566    /// for correct replay: matching the inline JSON conversion.
2567    #[test]
2568    fn context_replay_children_is_preserved() {
2569        let ops = vec![
2570            Operation::builder()
2571                .id("child-replay")
2572                .r#type(OperationType::Context)
2573                .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2574                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2575                .context_details(
2576                    aws_sdk_lambda::types::ContextDetails::builder()
2577                        .replay_children(true)
2578                        .build(),
2579                )
2580                .build()
2581                .unwrap(),
2582            Operation::builder()
2583                .id("child-inline")
2584                .r#type(OperationType::Context)
2585                .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2586                .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2587                .context_details(
2588                    aws_sdk_lambda::types::ContextDetails::builder()
2589                        .result(r#""child-result""#)
2590                        .build(),
2591                )
2592                .build()
2593                .unwrap(),
2594        ];
2595
2596        let log = operations_to_checkpoint_log(&ops);
2597
2598        let replayed = log.get("child-replay");
2599        assert!(replayed.is_some());
2600        let replayed = replayed.unwrap();
2601        assert!(
2602            replayed.replay_children,
2603            "ReplayChildren must survive the typed-state conversion"
2604        );
2605
2606        let inline = log.get("child-inline");
2607        assert!(inline.is_some());
2608        let inline = inline.unwrap();
2609        assert!(!inline.replay_children);
2610        assert_eq!(inline.result.as_deref(), Some(r#""child-result""#));
2611    }
2612
2613    /// Execution-type operations are filtered from the typed-state
2614    /// conversion, matching `parse_inline_operations` in the JSON path.
2615    #[test]
2616    fn execution_operations_are_filtered_from_log() {
2617        let ops = vec![
2618            Operation::builder()
2619                .id("exec-0")
2620                .r#type(OperationType::Execution)
2621                .status(aws_sdk_lambda::types::OperationStatus::Started)
2622                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2623                .build()
2624                .unwrap(),
2625            Operation::builder()
2626                .id("step-1")
2627                .r#type(OperationType::Step)
2628                .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2629                .start_timestamp(aws_smithy_types::DateTime::from_secs(1))
2630                .build()
2631                .unwrap(),
2632        ];
2633
2634        let log = operations_to_checkpoint_log(&ops);
2635
2636        assert!(
2637            log.get("exec-0").is_none(),
2638            "Execution operations must be filtered like the JSON path does"
2639        );
2640        assert!(log.get("step-1").is_some());
2641    }
2642
2643    // Get state
2644
2645    #[tokio::test]
2646    async fn get_state_returns_preloaded_operations() {
2647        let ops = vec![
2648            Operation::builder()
2649                .id("op-1")
2650                .r#type(OperationType::Step)
2651                .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2652                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2653                .build()
2654                .unwrap(),
2655        ];
2656
2657        let client = InMemoryExecutionClient::new(ops);
2658        let result = client.get_state("arn:test", "token-0", None).await;
2659
2660        assert!(result.is_ok());
2661        let output = result.unwrap();
2662        assert_eq!(output.operations.len(), 1);
2663        assert_eq!(output.operations.first().map(Operation::id), Some("op-1"));
2664    }
2665
2666    // Merge updated operations into log
2667
2668    #[test]
2669    fn merge_operations_inserts_callback_id_into_existing_log() {
2670        // Simulate: log starts with a STARTED callback (no callback_id yet).
2671        let log = CheckpointLog::from_records(vec![(
2672            "op-cb".to_owned(),
2673            CheckpointRecord {
2674                id: "op-cb".to_owned(),
2675                status: CheckpointStatus::Started,
2676                result: None,
2677                error_type: None,
2678                error_message: None,
2679                error_data: None,
2680                stack_trace: None,
2681                attempt: 0,
2682                invoke_result: None,
2683                invoke_error_type: None,
2684                invoke_error_message: None,
2685                invoke_error_data: None,
2686                invoke_stack_trace: None,
2687                replay_children: false,
2688                callback_id: None,
2689                op_type: None,
2690                sub_type: None,
2691                due_at: None,
2692                op_name: None,
2693                parent_id: None,
2694                start_timestamp: None,
2695                end_timestamp: None,
2696            },
2697        )]);
2698
2699        // Backend returns the operation with callback_id assigned.
2700        let updated_ops = vec![
2701            Operation::builder()
2702                .id("op-cb")
2703                .r#type(OperationType::Callback)
2704                .status(aws_sdk_lambda::types::OperationStatus::Pending)
2705                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2706                .callback_details(
2707                    aws_sdk_lambda::types::CallbackDetails::builder()
2708                        .callback_id("assigned-cb-id-42")
2709                        .build(),
2710                )
2711                .build()
2712                .unwrap(),
2713        ];
2714
2715        merge_operations_into_log(&log, &updated_ops);
2716
2717        let record = log.get("op-cb");
2718        assert!(record.is_some());
2719        let record = record.unwrap();
2720        assert_eq!(record.callback_id.as_deref(), Some("assigned-cb-id-42"));
2721        assert_eq!(record.status, CheckpointStatus::Pending);
2722    }
2723
2724    #[test]
2725    fn merge_operations_adds_new_record_to_log() {
2726        let log = CheckpointLog::empty();
2727
2728        // Backend returns a new operation not previously in the log.
2729        let updated_ops = vec![
2730            Operation::builder()
2731                .id("new-op")
2732                .r#type(OperationType::Step)
2733                .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2734                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2735                .step_details(
2736                    aws_sdk_lambda::types::StepDetails::builder()
2737                        .result(r#""merged result""#)
2738                        .build(),
2739                )
2740                .build()
2741                .unwrap(),
2742        ];
2743
2744        merge_operations_into_log(&log, &updated_ops);
2745
2746        let record = log.get("new-op");
2747        assert!(record.is_some());
2748        let record = record.unwrap();
2749        assert_eq!(record.status, CheckpointStatus::Succeeded);
2750        assert_eq!(record.result.as_deref(), Some(r#""merged result""#));
2751    }
2752
2753    /// An operation of type `ty` and status `status`, with a callback ID
2754    /// when `callback_id` is set.
2755    fn status_op(
2756        id: &str,
2757        ty: OperationType,
2758        status: aws_sdk_lambda::types::OperationStatus,
2759        callback_id: Option<&str>,
2760    ) -> Operation {
2761        let mut builder = Operation::builder()
2762            .id(id)
2763            .r#type(ty)
2764            .status(status)
2765            .start_timestamp(aws_smithy_types::DateTime::from_secs(0));
2766        if let Some(callback_id) = callback_id {
2767            builder = builder.callback_details(
2768                aws_sdk_lambda::types::CallbackDetails::builder()
2769                    .callback_id(callback_id)
2770                    .build(),
2771            );
2772        }
2773        builder.build().unwrap()
2774    }
2775
2776    /// The four merge cases in one call: a new record and a status change
2777    /// count, a field-only change does not, and the `EXECUTION` operation
2778    /// is skipped and never enters the log.
2779    #[test]
2780    fn merge_reports_status_changes() {
2781        use aws_sdk_lambda::types::OperationStatus as S;
2782        let log = CheckpointLog::from_records(vec![
2783            (
2784                "b".to_owned(),
2785                CheckpointRecord::with_status("b", CheckpointStatus::Started),
2786            ),
2787            (
2788                "c".to_owned(),
2789                CheckpointRecord::with_status("c", CheckpointStatus::Started),
2790            ),
2791        ]);
2792        let changed = merge_operations_into_log(
2793            &log,
2794            &[
2795                status_op("a", OperationType::Step, S::Started, None),
2796                status_op("b", OperationType::Callback, S::Started, Some("cb-1")),
2797                status_op("c", OperationType::Step, S::Succeeded, None),
2798                status_op("exec", OperationType::Execution, S::Started, None),
2799            ],
2800        );
2801        assert_eq!(changed, ["a", "c"]);
2802        assert_eq!(
2803            log.get("b").unwrap().callback_id.as_deref(),
2804            Some("cb-1"),
2805            "the field-only change is still merged"
2806        );
2807        assert!(!log.contains("exec"));
2808    }
2809
2810    /// Two statuses this SDK does not know differ when their tokens
2811    /// differ, and are equal when the tokens are equal.
2812    #[test]
2813    fn merge_compares_unknown_statuses_by_token() {
2814        use aws_sdk_lambda::types::OperationStatus as S;
2815        let log = CheckpointLog::from_records(vec![(
2816            "u".to_owned(),
2817            CheckpointRecord::with_status("u", CheckpointStatus::Unknown("A".to_owned())),
2818        )]);
2819        let op = |token: &str| status_op("u", OperationType::Step, S::from(token), None);
2820        assert_eq!(
2821            merge_operations_into_log(&log, &[op("A")]),
2822            Vec::<String>::new()
2823        );
2824        assert_eq!(merge_operations_into_log(&log, &[op("B")]), ["u"]);
2825    }
2826
2827    // Pagination tests
2828
2829    /// Helper: builds a Step operation with a result for pagination tests.
2830    fn make_step_op(id: &str, result: &str) -> Operation {
2831        Operation::builder()
2832            .id(id)
2833            .r#type(OperationType::Step)
2834            .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
2835            .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
2836            .step_details(
2837                aws_sdk_lambda::types::StepDetails::builder()
2838                    .result(result)
2839                    .build(),
2840            )
2841            .build()
2842            .unwrap()
2843    }
2844
2845    /// Tests that `get_state` in `InMemoryExecutionClient` returns all
2846    /// pre-loaded operations (simulating a paginated full fetch).
2847    #[tokio::test]
2848    async fn in_memory_get_state_returns_all_operations() {
2849        let ops = vec![
2850            make_step_op("step-1", "\"page1\""),
2851            make_step_op("step-2", "\"page2\""),
2852            make_step_op("step-3", "\"page3\""),
2853        ];
2854        let client = InMemoryExecutionClient::new(ops);
2855
2856        let state = client.get_state("arn:test", "token-0", None).await;
2857        assert!(state.is_ok());
2858        let state = state.unwrap();
2859        assert_eq!(state.operations.len(), 3);
2860    }
2861
2862    /// Tests that a paginated checkpoint response (with `next_marker`)
2863    /// triggers the caller to call `get_state`. This test verifies the
2864    /// `SuccessPaginated` variant works correctly.
2865    #[tokio::test]
2866    async fn checkpoint_with_marker_returns_paginated_response() {
2867        let all_ops = vec![
2868            make_step_op("step-1", "\"r1\""),
2869            make_step_op("step-2", "\"r2\""),
2870            make_step_op("step-3", "\"r3\""),
2871        ];
2872        let client = InMemoryExecutionClient::new(all_ops);
2873
2874        // Enqueue a checkpoint response that indicates pagination.
2875        let page1_ops = vec![make_step_op("step-1", "\"r1\"")];
2876        client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
2877            page1_ops,
2878            "marker-page-2".to_owned(),
2879        ));
2880
2881        let result = client.checkpoint("arn:test", "token-0", vec![]).await;
2882        assert!(result.is_ok());
2883        let output = result.unwrap();
2884        assert_eq!(output.updated_operations.len(), 1);
2885        assert_eq!(output.next_marker, Some("marker-page-2".to_owned()));
2886
2887        // Caller should then call get_state to get all operations.
2888        let full_state = client
2889            .get_state(
2890                "arn:test",
2891                output.checkpoint_token().unwrap_or_default(),
2892                None,
2893            )
2894            .await;
2895        assert!(full_state.is_ok());
2896        let full_state = full_state.unwrap();
2897        assert_eq!(full_state.operations.len(), 3);
2898    }
2899
2900    /// Tests that `operations_to_checkpoint_log` correctly converts a
2901    /// multi-page set of operations into a complete checkpoint log.
2902    #[test]
2903    fn operations_to_checkpoint_log_handles_multi_page_operations() {
2904        let ops = vec![
2905            make_step_op("step-1", "\"result-1\""),
2906            make_step_op("step-2", "\"result-2\""),
2907            make_step_op("step-3", "\"result-3\""),
2908        ];
2909
2910        let log = operations_to_checkpoint_log(&ops);
2911
2912        // All three operations should be in the log.
2913        assert!(log.get("step-1").is_some());
2914        assert!(log.get("step-2").is_some());
2915        assert!(log.get("step-3").is_some());
2916
2917        let r1 = log.get("step-1").unwrap();
2918        assert_eq!(r1.result.as_deref(), Some("\"result-1\""));
2919        assert_eq!(r1.status, CheckpointStatus::Succeeded);
2920
2921        let r3 = log.get("step-3").unwrap();
2922        assert_eq!(r3.result.as_deref(), Some("\"result-3\""));
2923    }
2924
2925    // Checkpoint error classification
2926
2927    use aws_sdk_lambda::config::http::HttpResponse;
2928    use aws_sdk_lambda::error::SdkError;
2929    use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError as CkptErr;
2930    use aws_smithy_runtime_api::http::StatusCode;
2931    use aws_smithy_types::body::SdkBody;
2932
2933    /// A raw HTTP response with the given status and an empty body.
2934    fn raw_response(status: u16) -> HttpResponse {
2935        HttpResponse::new(
2936            StatusCode::try_from(status).expect("valid HTTP status"),
2937            SdkBody::empty(),
2938        )
2939    }
2940
2941    /// Classifies `err` as a service error whose raw response has `status`.
2942    fn classify<E: Into<CkptErr>>(err: E, status: u16) -> ExecutionClientError {
2943        classify_checkpoint_error(SdkError::service_error(err.into(), raw_response(status)))
2944    }
2945
2946    fn invalid_parameter(message: Option<&str>) -> CkptErr {
2947        let mut builder = aws_sdk_lambda::types::error::InvalidParameterValueException::builder();
2948        if let Some(m) = message {
2949            builder = builder.message(m);
2950        }
2951        CkptErr::InvalidParameterValueException(builder.build())
2952    }
2953
2954    #[test]
2955    fn throttling_and_5xx_classify_retryable() {
2956        let throttled = classify(
2957            CkptErr::TooManyRequestsException(
2958                aws_sdk_lambda::types::error::TooManyRequestsException::builder().build(),
2959            ),
2960            429,
2961        );
2962        assert!(throttled.is_retryable());
2963
2964        let server = classify(
2965            CkptErr::ServiceException(
2966                aws_sdk_lambda::types::error::ServiceException::builder().build(),
2967            ),
2968            500,
2969        );
2970        assert!(server.is_retryable());
2971    }
2972
2973    #[test]
2974    fn invalid_parameter_classifies_non_retryable() {
2975        assert!(!classify(invalid_parameter(Some("payload too large")), 400).is_retryable());
2976        // No message at all: no stale-token evidence, permanent rejection.
2977        assert!(!classify(invalid_parameter(None), 400).is_retryable());
2978    }
2979
2980    /// The stale-token carve-out: `InvalidParameterValueException` whose
2981    /// message starts with `Invalid checkpoint token` resolves on
2982    /// re-invocation (the service issues a fresh token), so it is
2983    /// retryable. Prefix match only: a stale-token mention elsewhere in
2984    /// the message does not qualify. The carve-out precedes the status
2985    /// rule, so the 400 status does not make it non-retryable.
2986    #[test]
2987    fn stale_token_rejection_classifies_retryable() {
2988        let stale = classify(
2989            invalid_parameter(Some("Invalid checkpoint token: token has been superseded")),
2990            400,
2991        );
2992        assert!(stale.is_retryable());
2993
2994        let not_prefix = classify(
2995            invalid_parameter(Some(
2996                "field X rejected (not an Invalid checkpoint token case)",
2997            )),
2998            400,
2999        );
3000        assert!(!not_prefix.is_retryable());
3001
3002        let wrong_case = classify(
3003            invalid_parameter(Some("Invalid Checkpoint Token: old non-service spelling")),
3004            400,
3005        );
3006        assert!(!wrong_case.is_retryable());
3007    }
3008
3009    #[test]
3010    fn kms_misconfiguration_classifies_non_retryable() {
3011        let kms = classify(
3012            CkptErr::KmsAccessDeniedException(
3013                aws_sdk_lambda::types::error::KmsAccessDeniedException::builder().build(),
3014            ),
3015            502,
3016        );
3017        assert!(!kms.is_retryable());
3018    }
3019
3020    /// An unmodeled 4xx other than 429 is the service rejecting the
3021    /// request, so the same write fails the same way on every invocation:
3022    /// it fails the execution, as in JS.
3023    #[test]
3024    fn unhandled_4xx_classifies_non_retryable() {
3025        for status in [400, 403, 404, 409, 413] {
3026            let err = classify(CkptErr::unhandled("unmodeled rejection"), status);
3027            assert!(!err.is_retryable(), "status {status} must be non-retryable");
3028        }
3029    }
3030
3031    /// An unmodeled 429 is throttling, so it stays retryable.
3032    #[test]
3033    fn unhandled_429_classifies_retryable() {
3034        assert!(classify(CkptErr::unhandled("throttled"), 429).is_retryable());
3035    }
3036
3037    /// Unmodeled 5xx errors stay RETRYABLE. Under the #43 model a
3038    /// non-retryable classification fails the execution, so a novel
3039    /// transient error must not kill executions by default (the pre-#43
3040    /// non-retryable default relied on the old model, where the surfaced
3041    /// error only failed the invocation: r3726032098).
3042    #[test]
3043    fn unhandled_5xx_classifies_retryable() {
3044        for status in [500, 502, 503] {
3045            let err = classify(CkptErr::unhandled("some new service error"), status);
3046            assert!(err.is_retryable(), "status {status} must be retryable");
3047        }
3048    }
3049
3050    /// A 4xx response whose body the SDK could not parse is a service
3051    /// rejection too, so it fails the execution.
3052    #[test]
3053    fn response_error_4xx_classifies_non_retryable() {
3054        let err: SdkError<CkptErr, HttpResponse> =
3055            SdkError::response_error("unparseable body", raw_response(403));
3056        assert!(!classify_checkpoint_error(err).is_retryable());
3057    }
3058
3059    /// An unparseable 200 response is a channel fault, not a rejection.
3060    #[test]
3061    fn response_error_200_classifies_retryable() {
3062        let err: SdkError<CkptErr, HttpResponse> =
3063            SdkError::response_error("unparseable body", raw_response(200));
3064        assert!(classify_checkpoint_error(err).is_retryable());
3065    }
3066
3067    #[test]
3068    fn transport_failures_classify_retryable() {
3069        let timeout: SdkError<CkptErr, HttpResponse> = SdkError::timeout_error("connect timed out");
3070        assert!(classify_checkpoint_error(timeout).is_retryable());
3071    }
3072
3073    /// Canary: pins the variant set of `CheckpointDurableExecutionError`,
3074    /// the upstream enum [`classify_checkpoint_error`] matches with a
3075    /// wildcard arm.
3076    ///
3077    /// The enum is `#[non_exhaustive]`, so the classifier's `match` cannot
3078    /// reject new variants at compile time: a variant added by an
3079    /// aws-sdk-lambda upgrade would silently take the unknown→retryable
3080    /// default. This test parses the enum out of the dependency's own
3081    /// source (the exact version Cargo.lock pins, in the local cargo
3082    /// registry) and fails when the variant set changes, forcing the
3083    /// classification to be revisited instead of defaulted. The sibling
3084    /// audit for the `types` enums lives in
3085    /// `upstream_type_enum_variant_canary` (issue #45).
3086    #[test]
3087    fn checkpoint_error_variant_canary() {
3088        const CLASSIFIED_VARIANTS: &[&str] = &[
3089            "InvalidParameterValueException",
3090            "KmsAccessDeniedException",
3091            "KmsDisabledException",
3092            "KmsInvalidStateException",
3093            "KmsNotFoundException",
3094            "ServiceException",
3095            "TooManyRequestsException",
3096            "Unhandled",
3097        ];
3098
3099        let source = read_sdk_source("src/operation/checkpoint_durable_execution.rs");
3100        let mut actual = extract_enum_variants(&source, "CheckpointDurableExecutionError");
3101        actual.sort_unstable();
3102
3103        let mut expected: Vec<String> = CLASSIFIED_VARIANTS
3104            .iter()
3105            .map(|s| (*s).to_owned())
3106            .collect();
3107        expected.sort_unstable();
3108
3109        assert_eq!(
3110            actual, expected,
3111            "aws-sdk-lambda's CheckpointDurableExecutionError variant set changed. \
3112             Reclassify the new/removed variants in classify_checkpoint_error \
3113             (src/client.rs), then update this canary's pinned list."
3114        );
3115    }
3116
3117    /// Canary: pins the variant set of every upstream `types` enum the SDK
3118    /// matches with a wildcard arm: `OperationStatus`, `OperationType`,
3119    /// `OperationAction`, `ExecutionStatus`, and `EventType` (issue #45,
3120    /// extending the #43 pattern above).
3121    ///
3122    /// All five enums are `#[non_exhaustive]`, so a variant added by an
3123    /// aws-sdk-lambda upgrade would silently take each match's wildcard
3124    /// default: for `OperationStatus` that now means
3125    /// `CheckpointStatus::Unknown`, which fails replay rather than
3126    /// misclassifying, but the canary still forces the variant to be mapped
3127    /// explicitly instead of shipping through the fallback. The pinned
3128    /// lists include the generated enums' own `Unknown` carrier variant.
3129    #[test]
3130    #[expect(clippy::too_many_lines)] // reason: table-driven canary pins five enums' variant lists
3131    fn upstream_type_enum_variant_canary() {
3132        struct Pin {
3133            file: &'static str,
3134            enum_name: &'static str,
3135            variants: &'static [&'static str],
3136            wildcard_sites: &'static str,
3137        }
3138        let pins = [
3139            Pin {
3140                file: "src/types/_operation_status.rs",
3141                enum_name: "OperationStatus",
3142                variants: &[
3143                    "Cancelled",
3144                    "Failed",
3145                    "Pending",
3146                    "Ready",
3147                    "Started",
3148                    "Stopped",
3149                    "Succeeded",
3150                    "TimedOut",
3151                    "Unknown",
3152                ],
3153                wildcard_sites: "sdk_status_to_checkpoint_status (src/client.rs); \
3154                                 operation_status_str and operation_status_wire_str \
3155                                 (src/test_util.rs)",
3156            },
3157            Pin {
3158                file: "src/types/_operation_type.rs",
3159                enum_name: "OperationType",
3160                variants: &[
3161                    "Callback",
3162                    "ChainedInvoke",
3163                    "Context",
3164                    "Execution",
3165                    "Step",
3166                    "Unknown",
3167                    "Wait",
3168                ],
3169                wildcard_sites: "operation_type_str (src/tracing_layer.rs); \
3170                                 operation_type_str (src/test_util.rs)",
3171            },
3172            Pin {
3173                file: "src/types/_operation_action.rs",
3174                enum_name: "OperationAction",
3175                variants: &["Cancel", "Fail", "Retry", "Start", "Succeed", "Unknown"],
3176                wildcard_sites: "PendingTransitionEvent::emit (src/tracing_layer.rs); \
3177                                 BackendState::apply_update (src/test_util.rs); \
3178                                 child_starts_before_first_terminal \
3179                                 (src/map_parallel.rs tests)",
3180            },
3181            Pin {
3182                file: "src/types/_execution_status.rs",
3183                enum_name: "ExecutionStatus",
3184                variants: &[
3185                    "Failed",
3186                    "Running",
3187                    "Stopped",
3188                    "Succeeded",
3189                    "TimedOut",
3190                    "Unknown",
3191                ],
3192                wildcard_sites: "execution_status_str (src/test_util.rs)",
3193            },
3194            Pin {
3195                file: "src/types/_event_type.rs",
3196                enum_name: "EventType",
3197                variants: &[
3198                    "CallbackFailed",
3199                    "CallbackStarted",
3200                    "CallbackSucceeded",
3201                    "CallbackTimedOut",
3202                    "ChainedInvokeFailed",
3203                    "ChainedInvokeStarted",
3204                    "ChainedInvokeStopped",
3205                    "ChainedInvokeSucceeded",
3206                    "ChainedInvokeTimedOut",
3207                    "ContextFailed",
3208                    "ContextStarted",
3209                    "ContextSucceeded",
3210                    "ExecutionFailed",
3211                    "ExecutionStarted",
3212                    "ExecutionStopped",
3213                    "ExecutionSucceeded",
3214                    "ExecutionTimedOut",
3215                    "InvocationCompleted",
3216                    "StepFailed",
3217                    "StepStarted",
3218                    "StepSucceeded",
3219                    "Unknown",
3220                    "WaitCancelled",
3221                    "WaitStarted",
3222                    "WaitSucceeded",
3223                ],
3224                wildcard_sites: "apply_event_to_op and event_op_type (src/test_util.rs)",
3225            },
3226        ];
3227
3228        for pin in pins {
3229            let source = read_sdk_source(pin.file);
3230            let mut actual = extract_enum_variants(&source, pin.enum_name);
3231            actual.sort_unstable();
3232
3233            let mut expected: Vec<String> = pin.variants.iter().map(|s| (*s).to_owned()).collect();
3234            expected.sort_unstable();
3235
3236            assert_eq!(
3237                actual, expected,
3238                "aws-sdk-lambda's {} variant set changed. Classify the new/removed \
3239                 variants explicitly at every wildcard match site — {} — then update \
3240                 this canary's pinned list.",
3241                pin.enum_name, pin.wildcard_sites
3242            );
3243        }
3244    }
3245
3246    /// A modeled `OperationStatus` maps to its explicit `CheckpointStatus`;
3247    /// anything the SDK build does not model maps to
3248    /// `CheckpointStatus::Unknown` carrying the raw wire value, never to a
3249    /// guessed known status (the pre-#45 wildcard mapped it to `Started`,
3250    /// silently treating a possibly-terminal operation as re-runnable).
3251    #[test]
3252    fn unknown_operation_status_maps_to_unknown_carrying_raw() {
3253        use aws_sdk_lambda::types::OperationStatus;
3254
3255        let novel = OperationStatus::from("PAUSED");
3256        assert_eq!(
3257            sdk_status_to_checkpoint_status(&novel),
3258            CheckpointStatus::Unknown("PAUSED".to_owned())
3259        );
3260
3261        let modeled = [
3262            (OperationStatus::Started, CheckpointStatus::Started),
3263            (OperationStatus::Pending, CheckpointStatus::Pending),
3264            (OperationStatus::Ready, CheckpointStatus::Ready),
3265            (OperationStatus::Succeeded, CheckpointStatus::Succeeded),
3266            (OperationStatus::Failed, CheckpointStatus::Failed),
3267            (OperationStatus::Cancelled, CheckpointStatus::Cancelled),
3268            (OperationStatus::TimedOut, CheckpointStatus::TimedOut),
3269            (OperationStatus::Stopped, CheckpointStatus::Stopped),
3270        ];
3271        for (sdk, expected) in modeled {
3272            assert_eq!(
3273                sdk_status_to_checkpoint_status(&sdk),
3274                expected,
3275                "modeled variant {sdk:?} must map explicitly"
3276            );
3277        }
3278    }
3279
3280    /// Reads a source file out of the aws-sdk-lambda version Cargo.lock
3281    /// pins, from the local cargo registry (already extracted: the test
3282    /// binary that runs this compiled against it).
3283    fn read_sdk_source(relative: &str) -> String {
3284        let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
3285        let lockfile = std::fs::read_to_string(manifest_dir.join("Cargo.lock"))
3286            .expect("Cargo.lock readable at the workspace root");
3287        let version = lockfile
3288            .split("[[package]]")
3289            .find(|block| block.contains("name = \"aws-sdk-lambda\""))
3290            .and_then(|block| {
3291                block
3292                    .lines()
3293                    .find_map(|line| line.trim().strip_prefix("version = \""))
3294                    .map(|rest| rest.trim_end_matches('"').to_owned())
3295            })
3296            .expect("aws-sdk-lambda pinned in Cargo.lock");
3297
3298        let cargo_home = std::env::var_os("CARGO_HOME").map_or_else(
3299            || {
3300                let home = std::env::var_os("HOME").expect("HOME set");
3301                std::path::PathBuf::from(home).join(".cargo")
3302            },
3303            std::path::PathBuf::from,
3304        );
3305        let registry_src = cargo_home.join("registry").join("src");
3306        let crate_dir_name = format!("aws-sdk-lambda-{version}");
3307        let crate_dir = std::fs::read_dir(&registry_src)
3308            .expect("cargo registry src directory readable")
3309            .filter_map(Result::ok)
3310            .map(|entry| entry.path().join(&crate_dir_name))
3311            .find(|candidate| candidate.is_dir())
3312            .expect("pinned aws-sdk-lambda version extracted in the cargo registry");
3313
3314        std::fs::read_to_string(crate_dir.join(relative)).expect("SDK source file readable")
3315    }
3316
3317    /// Extracts the variant identifiers of a top-level `pub enum` from
3318    /// generated SDK source. The generated layout is stable: variants sit
3319    /// at one indentation level, as `Identifier(...)` or bare
3320    /// `Identifier,`.
3321    #[expect(clippy::panic, clippy::map_unwrap_or)]
3322    // reason: test helper assertions over generated-source text
3323    fn extract_enum_variants(source: &str, enum_name: &str) -> Vec<String> {
3324        let header = format!("pub enum {enum_name} {{");
3325        let start = source
3326            .find(&header)
3327            .unwrap_or_else(|| panic!("enum {enum_name} present in SDK source"));
3328        let body_start = start + header.len();
3329        let body_end = source[body_start..]
3330            .find("\n}")
3331            .map(|offset| body_start + offset)
3332            .unwrap_or_else(|| panic!("enum {enum_name} body terminated"));
3333        let body = &source[body_start..body_end];
3334
3335        body.lines()
3336            .filter_map(|line| {
3337                let trimmed = line.trim();
3338                // Variant lines start with an uppercase identifier followed
3339                // directly by `(` (tuple payload) or `,` (unit variant).
3340                // Doc comments, attributes (including multi-line
3341                // deprecation notes), and continuation lines do not.
3342                let first = trimmed.chars().next()?;
3343                if !first.is_ascii_uppercase() {
3344                    return None;
3345                }
3346                let ident: String = trimmed
3347                    .chars()
3348                    .take_while(char::is_ascii_alphanumeric)
3349                    .collect();
3350                let rest = trimmed.get(ident.len()..)?;
3351                if ident.is_empty() || !(rest.starts_with('(') || rest.starts_with(',')) {
3352                    return None;
3353                }
3354                Some(ident)
3355            })
3356            .collect()
3357    }
3358
3359    // User-Agent token
3360    //
3361    // These tests send requests through `LambdaExecutionClient` over a
3362    // `StaticReplayClient`, which records the headers of every attempt.
3363    // They build the expected token from the literal and the crate version,
3364    // not from `USER_AGENT_TOKEN`, so a wrong constant fails them.
3365
3366    use aws_sdk_lambda::config::{AppName, BehaviorVersion, Credentials, Region};
3367    use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
3368    use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
3369    use aws_smithy_types::retry::RetryConfig;
3370
3371    fn expected_token() -> String {
3372        format!(
3373            "aws-durable-execution-sdk-rust/{}",
3374            env!("CARGO_PKG_VERSION")
3375        )
3376    }
3377
3378    fn replay_event(status: u16, body: &'static str) -> ReplayEvent {
3379        let response = HttpResponse::new(
3380            StatusCode::try_from(status).expect("valid status"),
3381            SdkBody::from(body),
3382        );
3383        ReplayEvent::new(HttpRequest::new(SdkBody::empty()), response)
3384    }
3385
3386    /// Builds a `LambdaExecutionClient` over `http`, with a near-zero retry
3387    /// backoff so the retry test does not sleep.
3388    fn replay_lambda_client(
3389        http: &StaticReplayClient,
3390        app_name: Option<AppName>,
3391    ) -> LambdaExecutionClient {
3392        let mut config = aws_sdk_lambda::config::Builder::new()
3393            .behavior_version(BehaviorVersion::latest())
3394            .region(Region::new("us-east-1"))
3395            .credentials_provider(Credentials::new("akid", "secret", None, None, "test"))
3396            .retry_config(
3397                RetryConfig::standard().with_initial_backoff(std::time::Duration::from_millis(1)),
3398            )
3399            .http_client(http.clone());
3400        config.set_app_name(app_name);
3401        LambdaExecutionClient::new(aws_sdk_lambda::Client::from_conf(config.build()))
3402    }
3403
3404    /// The `User-Agent` values of every recorded attempt, one entry per
3405    /// header, so a request with two `User-Agent` headers is visible.
3406    fn user_agent_headers(http: &StaticReplayClient) -> Vec<Vec<String>> {
3407        http.actual_requests()
3408            .map(|req| {
3409                req.headers()
3410                    .get_all("user-agent")
3411                    .map(str::to_owned)
3412                    .collect()
3413            })
3414            .collect()
3415    }
3416
3417    /// Asserts criterion 1 on one attempt: exactly one `User-Agent` header,
3418    /// and exactly one of its space-separated parts equals the token.
3419    fn assert_one_token(headers: &[String]) {
3420        assert_eq!(
3421            headers.len(),
3422            1,
3423            "exactly one User-Agent header: {headers:?}"
3424        );
3425        let value = headers.first().expect("one header");
3426        let token = expected_token();
3427        let count = value.split(' ').filter(|part| *part == token).count();
3428        assert_eq!(count, 1, "exactly one {token} part in {value:?}");
3429    }
3430
3431    #[tokio::test]
3432    async fn user_agent_token_on_checkpoint() {
3433        let http = StaticReplayClient::new(vec![replay_event(200, r#"{"CheckpointToken":"t-1"}"#)]);
3434        let client = replay_lambda_client(&http, None);
3435
3436        let output = client
3437            .checkpoint("arn:test", "tok", Vec::new())
3438            .await
3439            .expect("replayed checkpoint succeeds");
3440        assert_eq!(output.checkpoint_token(), Some("t-1"));
3441
3442        let attempts = user_agent_headers(&http);
3443        assert_eq!(attempts.len(), 1);
3444        for headers in &attempts {
3445            assert_one_token(headers);
3446        }
3447    }
3448
3449    #[tokio::test]
3450    async fn user_agent_token_on_get_state() {
3451        let http = StaticReplayClient::new(vec![replay_event(200, r#"{"Operations":[]}"#)]);
3452        let client = replay_lambda_client(&http, None);
3453
3454        let output = client
3455            .get_state("arn:test", "tok", None)
3456            .await
3457            .expect("replayed state read succeeds");
3458        assert_eq!(output.next_marker(), None);
3459
3460        let attempts = user_agent_headers(&http);
3461        assert_eq!(attempts.len(), 1, "one get_state call sends one request");
3462        for headers in &attempts {
3463            assert_one_token(headers);
3464        }
3465    }
3466
3467    #[tokio::test]
3468    async fn user_agent_token_once_per_retry_attempt() {
3469        let http = StaticReplayClient::new(vec![
3470            replay_event(503, "{}"),
3471            replay_event(200, r#"{"CheckpointToken":"t-1"}"#),
3472        ]);
3473        let client = replay_lambda_client(&http, None);
3474
3475        client
3476            .checkpoint("arn:test", "tok", Vec::new())
3477            .await
3478            .expect("the retry succeeds");
3479
3480        let attempts = user_agent_headers(&http);
3481        assert_eq!(attempts.len(), 2, "the 503 is retried once");
3482        for headers in &attempts {
3483            assert_one_token(headers);
3484        }
3485    }
3486
3487    #[tokio::test]
3488    async fn user_agent_keeps_app_name() {
3489        let http = StaticReplayClient::new(vec![replay_event(200, r#"{"CheckpointToken":"t-1"}"#)]);
3490        let app_name = AppName::new("my-app").expect("valid app name");
3491        let client = replay_lambda_client(&http, Some(app_name));
3492
3493        client
3494            .checkpoint("arn:test", "tok", Vec::new())
3495            .await
3496            .expect("replayed checkpoint succeeds");
3497
3498        let token = expected_token();
3499        let requests: Vec<_> = http.actual_requests().collect();
3500        assert_eq!(requests.len(), 1);
3501        for req in requests {
3502            let user_agent = req.headers().get("user-agent").expect("User-Agent header");
3503            assert!(
3504                user_agent.split(' ').any(|part| part == "app/my-app"),
3505                "the app name stays in {user_agent:?}"
3506            );
3507            let amz_user_agent = req
3508                .headers()
3509                .get("x-amz-user-agent")
3510                .expect("x-amz-user-agent header");
3511            assert!(
3512                !amz_user_agent.contains(&token),
3513                "x-amz-user-agent stays unchanged: {amz_user_agent:?}"
3514            );
3515        }
3516    }
3517
3518    #[test]
3519    fn user_agent_with_token_appends_or_starts() {
3520        let token = expected_token();
3521        assert_eq!(
3522            user_agent_with_token(Some("aws-sdk-rust/1.3.0 os/linux")),
3523            format!("aws-sdk-rust/1.3.0 os/linux {token}")
3524        );
3525        assert_eq!(user_agent_with_token(None), token);
3526    }
3527
3528    /// A number with no leading zero.
3529    fn is_version_number(s: &str) -> bool {
3530        !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) && (s == "0" || !s.starts_with('0'))
3531    }
3532
3533    /// `X.Y.Z`, optionally followed by `-alpha`, `-beta` or `-rc` and an
3534    /// optional `.N`, with no leading zero in any number.
3535    fn has_supported_version_form(version: &str) -> bool {
3536        let (core, pre_release) = match version.split_once('-') {
3537            Some((core, pre_release)) => (core, Some(pre_release)),
3538            None => (version, None),
3539        };
3540        let numbers: Vec<&str> = core.split('.').collect();
3541        if numbers.len() != 3 || !numbers.iter().all(|n| is_version_number(n)) {
3542            return false;
3543        }
3544        let Some(pre_release) = pre_release else {
3545            return true;
3546        };
3547        let (label, number) = match pre_release.split_once('.') {
3548            Some((label, number)) => (label, Some(number)),
3549            None => (pre_release, None),
3550        };
3551        matches!(label, "alpha" | "beta" | "rc") && number.is_none_or(is_version_number)
3552    }
3553
3554    #[test]
3555    fn crate_version_has_the_supported_form() {
3556        for version in [
3557            "0.1.0",
3558            "1.0.0",
3559            "1.1.0-alpha",
3560            "1.1.0-beta.2",
3561            "1.1.0-rc.1",
3562        ] {
3563            assert!(has_supported_version_form(version), "{version} must fit");
3564        }
3565        for version in [
3566            "1.0.0-rc1",
3567            "1.0.0-preview",
3568            "1.0.0-RC.1",
3569            "1.0.0+build.1",
3570            "v1.0.0",
3571            "1.0",
3572            "01.0.0",
3573            "1.0.0-rc.01",
3574        ] {
3575            assert!(
3576                !has_supported_version_form(version),
3577                "{version} must not fit"
3578            );
3579        }
3580        assert!(
3581            has_supported_version_form(env!("CARGO_PKG_VERSION")),
3582            "the crate version {} has an unsupported form",
3583            env!("CARGO_PKG_VERSION")
3584        );
3585    }
3586}