Skip to main content

aws_durable_execution_sdk/
context.rs

1//! Context types for durable execution.
2//!
3//! [`DurableContext`] provides access to all durable operations.
4//! [`StepContext`] is passed to step bodies and deliberately omits durable
5//! operations: the type system enforces the "no nesting" rule at compile
6//! time.
7
8use std::future::Future;
9use std::sync::Arc;
10use std::time::Duration;
11
12use serde::{Serialize, de::DeserializeOwned};
13
14use crate::BoxError;
15use crate::builders::{
16    ChildBuilder, CreateCallbackBuilder, InvokeBuilder, JoinAllBuilder, MapBuilder,
17    ParallelBuilder, ParallelMixedBuilder, RaceBuilder, SelectOkBuilder, StepBuilder, Submitter,
18    TryJoinAllBuilder, WaitBuilder, WaitForCallbackBuilder, WaitForConditionBuilder,
19    WithRetryBuilder,
20};
21use crate::checkpoint_coalescer::{
22    Admission, BatchLimits, CheckpointBatch, CheckpointCoalescer, TrackedUpdate,
23    split_into_requests,
24};
25use crate::client::{
26    CheckpointOutput, ExecutionClient, ExecutionClientError, ExecutionClientErrorScope,
27};
28use crate::coordinator::{InvocationCoordinator, PermitKind};
29use crate::driver::{FatalError, FatalSeverity, TaskOwnership};
30use crate::engine::{
31    CheckpointLog, CheckpointRecord, CheckpointStatus, CheckpointStatusView, EngineState,
32    OperationId, TerminalReplaySnapshot,
33};
34use crate::error::{
35    NonDeterministicExecutionError, NonDeterministicExecutionErrorKind, OperationError,
36    OperationErrorKind, StepError, StepErrorKind,
37};
38use crate::future::{Branch, DurableFuture};
39
40use aws_sdk_lambda::types::{OperationAction, OperationUpdate};
41use tokio::sync::Mutex;
42
43/// The changes one checkpoint call merged, for `on_operation_change`.
44#[derive(Default)]
45struct ChangeSet {
46    /// The wire IDs of the log records whose status changed, each once, in
47    /// merge order.
48    ids: Vec<String>,
49    /// The `EXECUTION` operations that changed the plugins' slot.
50    executions: Vec<crate::plugin::OperationInfo>,
51}
52
53/// A checkpoint write that has already started, and the future of its
54/// result.
55///
56/// [`DurableContext::submit_checkpoint_updates`] starts the write before it
57/// returns: the updates are admitted to the buffer. Awaiting the value only
58/// waits for the result. Dropping it does not cancel the write, because the
59/// batch flusher does not depend on the caller. An at-least-once step uses
60/// this to run its body while its `START` write is in flight.
61pub(crate) struct SubmittedWrite {
62    result: std::pin::Pin<
63        Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send>,
64    >,
65    /// Set on the first poll (see [`SubmissionRecorder`]).
66    #[cfg(test)]
67    first_poll: Arc<std::sync::atomic::AtomicBool>,
68    /// The plugins and the log whose queued change sets the write delivers
69    /// when its result is ready (see [`Self::reported_by`]). `None` without
70    /// plugins, and once delivered.
71    report: Option<Box<WriteReport>>,
72}
73
74/// What a [`SubmittedWrite`] needs to deliver the change sets through its
75/// own before it returns its result.
76struct WriteReport {
77    plugins: Arc<crate::plugin::InvocationPlugins>,
78    log: Arc<CheckpointLog>,
79    /// The result, kept while the write waits for the deliveries.
80    ready: Option<Result<CheckpointOutput, ExecutionClientError>>,
81    /// The write's wait for the deliveries (see
82    /// [`InvocationPlugins::poll_changes_delivered`](crate::plugin::InvocationPlugins::poll_changes_delivered)).
83    wait: crate::plugin::DeliveryWait,
84}
85
86impl Drop for WriteReport {
87    /// A write dropped while it waits removes its waker registration.
88    fn drop(&mut self) {
89        self.plugins.cancel_wait(&mut self.wait);
90    }
91}
92
93impl SubmittedWrite {
94    fn new(
95        result: impl Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + 'static,
96    ) -> Self {
97        Self {
98            result: Box::pin(result),
99            #[cfg(test)]
100            first_poll: Arc::new(std::sync::atomic::AtomicBool::new(false)),
101            report: None,
102        }
103    }
104
105    /// Makes the write deliver the queued change sets of `ctx`'s plugins
106    /// to `on_operation_change` once its result is ready, `Ok` or `Err`, on
107    /// the task that polls it to completion. A write dropped before that
108    /// delivers nothing: another drain point delivers its change set. Does
109    /// nothing without plugins.
110    fn reported_by(mut self, ctx: &DurableContext) -> Self {
111        self.report = ctx.shared_plugins().map(|plugins| {
112            Box::new(WriteReport {
113                plugins,
114                log: Arc::clone(&ctx.inner.engine.checkpoint_log),
115                ready: None,
116                wait: crate::plugin::DeliveryWait::default(),
117            })
118        });
119        self
120    }
121
122    /// A write whose result `receiver` delivers. A test builds the retained
123    /// `START` of an operation with it.
124    #[cfg(test)]
125    pub(crate) fn from_receiver(
126        receiver: tokio::sync::oneshot::Receiver<Result<CheckpointOutput, ExecutionClientError>>,
127    ) -> Self {
128        Self::new(async move {
129            match receiver.await {
130                Ok(result) => result,
131                Err(_closed) => std::future::pending().await,
132            }
133        })
134    }
135
136    /// A rejected write: nothing is sent, and the result never resolves.
137    /// The rejected caller's task is being cancelled (see [`Admission`]).
138    fn rejected() -> Self {
139        Self::new(std::future::pending())
140    }
141}
142
143impl Future for SubmittedWrite {
144    type Output = Result<CheckpointOutput, ExecutionClientError>;
145
146    /// Returns the result once the change sets are delivered.
147    fn poll(
148        self: std::pin::Pin<&mut Self>,
149        cx: &mut std::task::Context<'_>,
150    ) -> std::task::Poll<Self::Output> {
151        let this = self.get_mut();
152        #[cfg(test)]
153        this.first_poll
154            .store(true, std::sync::atomic::Ordering::SeqCst);
155        let Some(report) = this.report.as_mut() else {
156            return this.result.as_mut().poll(cx);
157        };
158        let result = match report.ready.take() {
159            Some(result) => result,
160            None => std::task::ready!(this.result.as_mut().poll(cx)),
161        };
162        if report
163            .plugins
164            .poll_changes_delivered(&report.log, &mut report.wait, cx)
165            .is_pending()
166        {
167            report.ready = Some(result);
168            return std::task::Poll::Pending;
169        }
170        this.report = None;
171        std::task::Poll::Ready(result)
172    }
173}
174
175/// A checkpoint write whose position in the write order is fixed, but
176/// that is not sent yet.
177///
178/// [`DurableContext::reserve_checkpoint_updates`] puts the updates into the
179/// coalescer's open batch before it returns. A batch that holds a reserved
180/// write is not flushed, and a later submission joins that batch, so it
181/// cannot reach the service first. [`Self::release`] makes the write
182/// eligible to be sent and returns its result future. Dropping the value
183/// releases it too.
184///
185/// The start step of an operation whose JS order is "queue START, then
186/// fire `on_operation_start`" reserves START, fires the hook, then
187/// releases START. So the hook returns before the START request is sent,
188/// on any runtime, as in JS.
189pub(crate) struct ReservedWrite {
190    write: SubmittedWrite,
191    /// The coalescer whose open batch the write holds. `None` once
192    /// released, and for a rejected write.
193    hold: Option<Arc<CheckpointCoalescer>>,
194}
195
196impl ReservedWrite {
197    /// A rejected reservation: nothing is held, and the result never
198    /// resolves (see [`SubmittedWrite::rejected`]).
199    fn rejected() -> Self {
200        Self {
201            write: SubmittedWrite::rejected(),
202            hold: None,
203        }
204    }
205
206    /// Makes the write eligible to be sent, and returns its result future.
207    /// It does not wait for the write.
208    pub(crate) fn release(mut self) -> SubmittedWrite {
209        // `self` is dropped when this returns, and the drop releases the
210        // hold.
211        std::mem::replace(&mut self.write, SubmittedWrite::rejected())
212    }
213}
214
215impl Drop for ReservedWrite {
216    /// A dropped reservation is released, so the writes queued behind it
217    /// are not blocked forever.
218    fn drop(&mut self) {
219        if let Some(hold) = self.hold.take() {
220            hold.release_hold();
221        }
222    }
223}
224
225/// The test hook of the checkpoint writer: for a request's updates it
226/// returns the `Notify` the writer awaits before it takes the checkpoint
227/// token, or `None` to write at once.
228#[cfg(test)]
229pub(crate) type PreLockGate =
230    Arc<dyn Fn(&[OperationUpdate]) -> Option<Arc<tokio::sync::Notify>> + Send + Sync>;
231
232/// The test hook that sees every admitted submission: its updates and the
233/// first-poll flag of its [`SubmittedWrite`]. It runs before the write can
234/// be sent: an ordinary write calls it before it joins the coalescer, and a
235/// reservation calls it while its batch is held.
236#[cfg(test)]
237pub(crate) type SubmissionRecorder =
238    Arc<dyn Fn(&[OperationUpdate], &Arc<std::sync::atomic::AtomicBool>) + Send + Sync>;
239
240/// The test hook that each child-backed outcome path calls with the
241/// operation's wire ID and the action it is about to write, just before it
242/// awaits the operation's retained `START` write.
243#[cfg(test)]
244pub(crate) type SettlerEntryRecorder = Arc<dyn Fn(&str, OperationAction) + Send + Sync>;
245
246/// The test hooks shared by every context of the invocation.
247#[cfg(test)]
248#[derive(Default)]
249pub(crate) struct TestHooks {
250    gate: Option<PreLockGate>,
251    submissions: Option<SubmissionRecorder>,
252    settler_entries: Option<SettlerEntryRecorder>,
253}
254
255/// The result future of a write that joined `batch`: it resolves when the
256/// batch's flusher publishes the result.
257fn batch_result(batch: Arc<CheckpointBatch>) -> SubmittedWrite {
258    SubmittedWrite::new(async move {
259        loop {
260            // `enable()` registers the waiter BEFORE the result check, so a
261            // publish landing between the check and the await still wakes
262            // us.
263            let mut notified = std::pin::pin!(batch.notified());
264            notified.as_mut().enable();
265            if let Some(result) = batch.result_clone() {
266                return result;
267            }
268            notified.await;
269        }
270    })
271}
272
273/// The fatal error a failed checkpoint write records: a retryable failure
274/// fails the invocation with the message of
275/// [`checkpoint_invocation_failure_message`](crate::error::checkpoint_invocation_failure_message),
276/// and a non-retryable failure fails the execution with the record of
277/// [`checkpoint_execution_failure_wire`](crate::error::checkpoint_execution_failure_wire).
278fn checkpoint_failure_fatal(err: &ExecutionClientError) -> FatalError {
279    if err.is_retryable() {
280        FatalError {
281            error_type: crate::error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE.to_owned(),
282            error_message: crate::error::checkpoint_invocation_failure_message(err),
283            severity: FatalSeverity::FailInvocation,
284        }
285    } else {
286        let wire = crate::error::checkpoint_execution_failure_wire(err);
287        FatalError {
288            error_type: wire.error_type().unwrap_or_default().to_owned(),
289            error_message: wire.error_message().unwrap_or_default().to_owned(),
290            severity: FatalSeverity::FailExecution,
291        }
292    }
293}
294
295/// Why a checkpoint write returned no output.
296#[derive(Debug)]
297pub(crate) enum WriteError {
298    /// The request failed with this error, and the failure is recorded
299    /// (see [`InvocationCoordinator::fail_write`]).
300    Failed(ExecutionClientError),
301    /// Nothing more is sent: the response had no token, the token was
302    /// withdrawn before the request was sent (see
303    /// [`InvocationCoordinator::withdraw`]), or an earlier write failed.
304    /// The write's callers never resolve.
305    Stopped,
306}
307
308/// The user-code body the SDK is polling on the current task.
309///
310/// [`DurableContext::claim_operation_id`] compares it with the context an
311/// operation is called on.
312#[derive(Debug, Clone)]
313pub(crate) enum ActiveBody {
314    /// A body that runs on a `DurableContext`: the handler or a child body.
315    /// Holds that context's positional prefix ("" for the root).
316    Context(String),
317    /// A body that may not call durable operations: a step body or a
318    /// `wait_for_condition` check. Holds the operation's positional ID.
319    Step(String),
320    /// Plugin code: a wrap, or a start or end hook and its future. A
321    /// plugin observes operations, so a durable operation it called would
322    /// claim an ID that the handler's code did not create, and replay
323    /// would diverge.
324    Plugin,
325    /// The SDK derives the recorded error of a user function's outcome:
326    /// the error's `Display` and `source` run, which are user code. Holds
327    /// the operation's positional ID. An operation called there would
328    /// claim an ID on a live run only, since replay returns the recorded
329    /// error without deriving it again.
330    Outcome(String),
331    /// No body is active: the check does nothing. A plugin wrap's `next`
332    /// sets it when the code around the wrap had no value, so the code
333    /// inside sees exactly what it saw without the wrap.
334    Unset,
335}
336
337/// The body active now, or [`ActiveBody::Unset`] when none is.
338pub(crate) fn current_body() -> ActiveBody {
339    ACTIVE_BODY
340        .try_with(ActiveBody::clone)
341        .unwrap_or(ActiveBody::Unset)
342}
343
344tokio::task_local! {
345    /// The body the SDK is polling. `LocalKey::scope` holds the value only
346    /// while the wrapped future is polled. So a parent future joined with a
347    /// child body on one task does not see the child's value, and a task
348    /// the SDK spawns starts without a value.
349    static ACTIVE_BODY: ActiveBody;
350}
351
352/// Polls `fut` with `body` as the active body.
353///
354/// Callers pass a future that calls the user closure inside it, so code
355/// the closure runs before it returns its future is checked too.
356pub(crate) fn run_body<F: Future>(body: ActiveBody, fut: F) -> impl Future<Output = F::Output> {
357    ACTIVE_BODY.scope(body, fut)
358}
359
360/// Calls `f` with `body` as the active body.
361///
362/// A step clones the user closure before each attempt. The clone runs the
363/// captures' `Clone` implementations, which are user code. A terminal
364/// record replays without cloning. So an operation claimed inside a clone
365/// would claim an ID on a live attempt only. This scope makes such a claim
366/// fail the execution with a context usage error instead.
367pub(crate) fn run_body_sync<R>(body: ActiveBody, f: impl FnOnce() -> R) -> R {
368    ACTIVE_BODY.sync_scope(body, f)
369}
370
371/// The time left from `now` until `deadline_ms` (milliseconds since the
372/// Unix epoch): `None` when the deadline is 0, `Duration::ZERO` once it
373/// passed.
374fn remaining_until(deadline_ms: u64, now: std::time::SystemTime) -> Option<Duration> {
375    if deadline_ms == 0 {
376        return None;
377    }
378    let deadline = std::time::UNIX_EPOCH + Duration::from_millis(deadline_ms);
379    Some(deadline.duration_since(now).unwrap_or(Duration::ZERO))
380}
381
382/// Shows a context prefix in a context usage error: `root` for the root.
383pub(crate) fn display_prefix(prefix: &str) -> &str {
384    if prefix.is_empty() { "root" } else { prefix }
385}
386
387/// Returns the wire ID and payload of every `wait_for_condition` `RETRY`
388/// among `updates`, in order.
389fn wfc_retries(updates: &[OperationUpdate]) -> Vec<(String, Option<String>)> {
390    updates
391        .iter()
392        .filter(|update| {
393            *update.action() == OperationAction::Retry
394                && update.sub_type() == Some(crate::wait_for_condition::WFC_SUB_TYPE)
395        })
396        .map(|update| (update.id().to_owned(), update.payload().map(str::to_owned)))
397        .collect()
398}
399
400/// Marks the record of `wire_id` `PENDING` with `payload` as its result,
401/// after a `wait_for_condition` `RETRY` write whose response did not carry
402/// the operation.
403///
404/// The service moves a retried operation to `PENDING`. A response that
405/// does not carry the operation leaves the log at the status from before
406/// the write. A waiting operation that read that status would take it for
407/// the next status and run its next check at once. So the log takes the
408/// status the service recorded. A record already `PENDING` is kept, and a
409/// missing record stays missing. A response that carries the operation is
410/// merged instead, so its status is kept.
411fn mark_retry_pending(log: &CheckpointLog, wire_id: &str, payload: Option<&str>) {
412    log.update(wire_id, |record| {
413        if record.status != CheckpointStatus::Pending {
414            record.status = CheckpointStatus::Pending;
415            record.result = payload.map(str::to_owned);
416        }
417    });
418}
419
420/// Shared inner state for a durable execution context.
421struct Inner {
422    execution_arn: String,
423    lambda_context: lambda_runtime::Context,
424    /// Engine state (ID counter + checkpoint log) for this context's
425    /// namespace.
426    engine: Arc<EngineState>,
427    /// The invocation coordinator, shared by every context of the
428    /// invocation and by the driver.
429    coordinator: Arc<InvocationCoordinator>,
430    /// Task-ownership detector: catches user `tokio::spawn` misuse.
431    task_ownership: Arc<TaskOwnership>,
432    /// Execution client for checkpointing (None in test contexts without a client).
433    execution_client: Option<Arc<dyn ExecutionClient>>,
434    /// Mutable checkpoint token rotated on each checkpoint call.
435    /// Uses `tokio::sync::Mutex` (not `std::sync::Mutex`) because the lock
436    /// must be held across `await` points in [`DurableContext::checkpoint_updates`],
437    /// serializing all concurrent checkpoint callers through one critical section.
438    checkpoint_token: Arc<Mutex<String>>,
439    /// Checkpoint-write coalescer. Every checkpoint write goes through its
440    /// writer: a write made while the drive runs joins its buffer, and the
441    /// write after the drive takes its writer lock. Shared with every child
442    /// context so updates from all namespaces coalesce into the same
443    /// batches. Its window is zero unless [`Options`](crate::Options)
444    /// configured a `checkpoint_delay`.
445    coalescer: Arc<CheckpointCoalescer>,
446    /// The test hooks, shared by the root and every child context (see
447    /// [`TestHooks`]).
448    #[cfg(test)]
449    hooks: Arc<std::sync::Mutex<TestHooks>>,
450    /// Cached parent wire ID: the SHA-256 hash of this context's prefix
451    /// (positional ID of the parent operation). `None` for root contexts.
452    parent_wire_id: Option<String>,
453    /// Abandoned owners of this invocation, shared by the root and every
454    /// context derived from it (see [`Admission`]).
455    admission: Arc<Admission>,
456    /// The owners this context runs in, outermost first: the wire IDs of
457    /// the non-virtual child contexts (batch items included) that enclose
458    /// it. Empty at the root. A checkpoint update from this context is
459    /// admitted only while none of them is abandoned.
460    owners: Arc<[String]>,
461    /// This namespace's `durable_execution` span. For a root context it is
462    /// the handler-level span wrapping the invocation; for a child context
463    /// (sequential child, map/parallel branch, callback body), recorded,
464    /// virtual or flat, it is a child of the span current when the context
465    /// is created, wrapping that child's body. [`DurableContext::mint_id`]
466    /// re-records its `isReplay` field after every operation claim minted
467    /// through this context, so replay-aware filters (see
468    /// `tracing_layer::ReplayFilterLayer`) can suppress log events emitted
469    /// while THIS namespace is replaying: the filter reads the SDK spans
470    /// from an event up to and including its nearest namespace span, and
471    /// stops there. Per-namespace spans are what keep
472    /// concurrent branches, each with its own replay high-water mark, from
473    /// clobbering each other's flag.
474    replay_span: tracing::Span,
475    /// The plugins of the invocation. The root context of a wrapped
476    /// invocation holds them, and every context derived from it shares
477    /// the same `Arc`. `None` without a registered plugin factory.
478    plugins: Option<Arc<crate::plugin::InvocationPlugins>>,
479    /// How many context levels this context sits below the root: 0 at the
480    /// root, plus 1 for each [`DurableContext::new_child`]. A Flat batch
481    /// item's context keeps the batch's level, and a virtual child's context
482    /// keeps its caller's level.
483    nesting_level: u32,
484    /// The invocation's
485    /// [`plugin_child_operations_depth`](crate::OptionsBuilder::plugin_child_operations_depth),
486    /// copied into every derived context.
487    preserve_child_depth: u32,
488    /// True inside a body that runs again to rebuild the result of a
489    /// recorded success (`ReplayChildren`). Copied into every derived context.
490    rebuilding: bool,
491}
492
493impl std::fmt::Debug for Inner {
494    /// Hand-written to keep `checkpoint_token`, a credential-like value,
495    /// out of log output, and to skip the engine/client/signal internals
496    /// that make a derived impl unreadable. See issue #30.
497    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
498        f.debug_struct("Inner")
499            .field("execution_arn", &self.execution_arn)
500            .field("parent_wire_id", &self.parent_wire_id)
501            .field("is_replaying", &self.engine.is_replaying())
502            .field("checkpoint_token", &"<redacted>")
503            .finish_non_exhaustive()
504    }
505}
506
507/// The durable execution context: a cheap-to-clone handle providing access
508/// to all durable operations.
509///
510/// `DurableContext` is `Clone + Send + Sync` (backed by an `Arc`). Clone it
511/// freely to share across async boundaries. Every durable operation method
512/// except [`with_retry`](Self::with_retry) and the four combinators claims
513/// its operation ID synchronously at the call site: polling order never
514/// affects identity. A `with_retry` block claims its ID when its builder
515/// becomes its future (`.await`, `.future()` or `.spawn()`). A combinator
516/// ([`try_join_all`](Self::try_join_all), [`join_all`](Self::join_all),
517/// [`select_ok`](Self::select_ok) and [`race`](Self::race)) claims its ID
518/// when it is first awaited or polled, or when it is spawned. A combinator
519/// that is dropped before that claims no ID and records nothing. A
520/// combinator passed to another combinator must already have claimed its
521/// ID: start it with `.spawn()` and pass the returned handle. A
522/// still-unclaimed combinator input fails the execution with a context
523/// usage error.
524///
525/// # Examples
526///
527/// ```no_run
528/// use aws_durable_execution_sdk as durable;
529///
530/// async fn handler(
531///     _event: serde_json::Value,
532///     ctx: durable::DurableContext,
533/// ) -> Result<String, durable::BoxError> {
534///     // Access execution metadata
535///     let arn = ctx.execution_arn();
536///     let replaying = ctx.is_replaying();
537///
538///     // Perform a durable step
539///     let result = ctx.step(|_| async { Ok("done".to_owned()) })
540///         .name("example")
541///         .await?;
542///     Ok(result)
543/// }
544/// ```
545#[derive(Clone)]
546#[non_exhaustive]
547pub struct DurableContext {
548    inner: Arc<Inner>,
549    /// The first-poll flag of the handle an operation started on its own
550    /// task returns. `None` on every context except the clone that starts
551    /// such an operation (see [`Self::with_awaited`]). Every wait
552    /// registration of an operation run on that clone carries the flag, so
553    /// the refresh scheduler skips it until the handle is first polled.
554    awaited: Option<Arc<std::sync::atomic::AtomicBool>>,
555}
556
557impl std::fmt::Debug for DurableContext {
558    /// Hand-written so `tracing::debug!(?ctx)` in a handler cannot leak the
559    /// checkpoint token into `CloudWatch` Logs: the token prints as
560    /// `"<redacted>"` and engine/client internals are skipped. See issue #30.
561    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
562        f.debug_struct("DurableContext")
563            .field("execution_arn", &self.inner.execution_arn)
564            .field("parent_wire_id", &self.inner.parent_wire_id)
565            .field("is_replaying", &self.inner.engine.is_replaying())
566            .field("checkpoint_token", &"<redacted>")
567            .finish_non_exhaustive()
568    }
569}
570
571impl DurableContext {
572    /// Builds a context for unit tests: a root context with an empty
573    /// checkpoint log and no execution client.
574    #[cfg(test)]
575    pub(crate) fn test_context() -> Self {
576        let execution_arn = String::from("arn:aws:lambda:us-east-1:123456789012:function:test");
577        let lambda_context = lambda_runtime::Context::default();
578        let engine = Arc::new(EngineState::new_root(Arc::new(CheckpointLog::empty())));
579        let replay_span = crate::tracing_layer::execution_span(
580            &execution_arn,
581            &lambda_context.request_id,
582            lambda_context.tenant_id.as_deref(),
583            engine.is_replaying(),
584        );
585        Self {
586            inner: Arc::new(Inner {
587                execution_arn,
588                lambda_context,
589                engine,
590                coordinator: InvocationCoordinator::new(),
591                task_ownership: Arc::new(TaskOwnership::new_current()),
592                execution_client: None,
593                checkpoint_token: Arc::new(Mutex::new(String::new())),
594                coalescer: Arc::new(CheckpointCoalescer::new(Duration::ZERO)),
595                #[cfg(test)]
596                hooks: Arc::default(),
597                parent_wire_id: None,
598                admission: Arc::new(Admission::new()),
599                owners: Arc::from(Vec::new()),
600                replay_span,
601                plugins: None,
602                nesting_level: 0,
603                preserve_child_depth: 0,
604                rebuilding: false,
605            }),
606            awaited: None,
607        }
608    }
609
610    /// Creates a root context with the given execution state (test-only:
611    /// production roots are built by the handler wrapper with a client and
612    /// token via [`Self::new_root_with_client`]-equivalent wiring in
613    /// `lib.rs`).
614    #[cfg(test)]
615    pub(crate) fn new_root(
616        execution_arn: String,
617        lambda_context: lambda_runtime::Context,
618        checkpoint_log: Arc<CheckpointLog>,
619    ) -> Self {
620        let engine = Arc::new(EngineState::new_root(checkpoint_log));
621        let replay_span = crate::tracing_layer::execution_span(
622            &execution_arn,
623            &lambda_context.request_id,
624            lambda_context.tenant_id.as_deref(),
625            engine.is_replaying(),
626        );
627        Self {
628            inner: Arc::new(Inner {
629                execution_arn,
630                lambda_context,
631                engine,
632                coordinator: InvocationCoordinator::new(),
633                task_ownership: Arc::new(TaskOwnership::new_current()),
634                execution_client: None,
635                checkpoint_token: Arc::new(Mutex::new(String::new())),
636                coalescer: Arc::new(CheckpointCoalescer::new(Duration::ZERO)),
637                #[cfg(test)]
638                hooks: Arc::default(),
639                parent_wire_id: None,
640                admission: Arc::new(Admission::new()),
641                owners: Arc::from(Vec::new()),
642                replay_span,
643                plugins: None,
644                nesting_level: 0,
645                preserve_child_depth: 0,
646                rebuilding: false,
647            }),
648            awaited: None,
649        }
650    }
651
652    /// Creates a root context with a client and token (test-only harness
653    /// for exercising live-path operation execution against a test double).
654    #[cfg(test)]
655    pub(crate) fn new_root_with_client(
656        execution_arn: String,
657        lambda_context: lambda_runtime::Context,
658        checkpoint_log: Arc<CheckpointLog>,
659        client: Arc<dyn ExecutionClient>,
660        checkpoint_token: String,
661    ) -> Self {
662        let engine = Arc::new(EngineState::new_root(checkpoint_log));
663        let replay_span = crate::tracing_layer::execution_span(
664            &execution_arn,
665            &lambda_context.request_id,
666            lambda_context.tenant_id.as_deref(),
667            engine.is_replaying(),
668        );
669        Self {
670            inner: Arc::new(Inner {
671                execution_arn,
672                lambda_context,
673                engine,
674                coordinator: InvocationCoordinator::new(),
675                task_ownership: Arc::new(TaskOwnership::new_current()),
676                execution_client: Some(client),
677                checkpoint_token: Arc::new(Mutex::new(checkpoint_token)),
678                coalescer: Arc::new(CheckpointCoalescer::new(Duration::ZERO)),
679                #[cfg(test)]
680                hooks: Arc::default(),
681                parent_wire_id: None,
682                admission: Arc::new(Admission::new()),
683                owners: Arc::from(Vec::new()),
684                replay_span,
685                plugins: None,
686                nesting_level: 0,
687                preserve_child_depth: 0,
688                rebuilding: false,
689            }),
690            awaited: None,
691        }
692    }
693
694    /// Creates a root context with a client, token, and checkpoint
695    /// buffering window threaded in from
696    /// [`Options`](crate::Options) (internal). `checkpoint_buffer_window`
697    /// is `Duration::ZERO` by default (no added delay; writes batch behind
698    /// the single-writer lock), and the `checkpoint_delay` coalescing window
699    /// when one is configured.
700    /// `preserve_child_depth` is the
701    /// [`plugin_child_operations_depth`](crate::OptionsBuilder::plugin_child_operations_depth)
702    /// of the invocation.
703    #[expect(clippy::too_many_arguments, reason = "one argument per root setting")]
704    pub(crate) fn new_root_with_client_and_defaults(
705        execution_arn: String,
706        lambda_context: lambda_runtime::Context,
707        checkpoint_log: Arc<CheckpointLog>,
708        client: Arc<dyn ExecutionClient>,
709        checkpoint_token: String,
710        checkpoint_buffer_window: Duration,
711        preserve_child_depth: u32,
712        plugins: Option<Arc<crate::plugin::InvocationPlugins>>,
713    ) -> Self {
714        let engine = Arc::new(EngineState::new_root(checkpoint_log));
715        let replay_span = crate::tracing_layer::execution_span(
716            &execution_arn,
717            &lambda_context.request_id,
718            lambda_context.tenant_id.as_deref(),
719            engine.is_replaying(),
720        );
721        Self {
722            inner: Arc::new(Inner {
723                execution_arn,
724                lambda_context,
725                engine,
726                coordinator: InvocationCoordinator::new(),
727                task_ownership: Arc::new(TaskOwnership::new_current()),
728                execution_client: Some(client),
729                checkpoint_token: Arc::new(Mutex::new(checkpoint_token)),
730                coalescer: Arc::new(CheckpointCoalescer::new(checkpoint_buffer_window)),
731                #[cfg(test)]
732                hooks: Arc::default(),
733                parent_wire_id: None,
734                admission: Arc::new(Admission::new()),
735                owners: Arc::from(Vec::new()),
736                replay_span,
737                plugins,
738                nesting_level: 0,
739                preserve_child_depth,
740                rebuilding: false,
741            }),
742            awaited: None,
743        }
744    }
745
746    /// Test-only root with an empty log, an in-memory client and the
747    /// plugin child operations depth `depth`, built through
748    /// [`Self::new_root_with_client_and_defaults`]. Returns the client, whose
749    /// recorded updates a test reads.
750    #[cfg(test)]
751    pub(crate) fn test_context_with_depth(
752        depth: u32,
753    ) -> (Self, Arc<crate::client::InMemoryExecutionClient>) {
754        let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
755        let ctx = Self::test_context_with_depth_and_client(
756            depth,
757            Arc::clone(&client) as Arc<dyn ExecutionClient>,
758        );
759        (ctx, client)
760    }
761
762    /// [`Self::test_context_with_depth`] with a caller-supplied client.
763    #[cfg(test)]
764    pub(crate) fn test_context_with_depth_and_client(
765        depth: u32,
766        client: Arc<dyn ExecutionClient>,
767    ) -> Self {
768        Self::new_root_with_client_and_defaults(
769            "arn:test".to_owned(),
770            lambda_runtime::Context::default(),
771            Arc::new(CheckpointLog::empty()),
772            client,
773            "token0".to_owned(),
774            Duration::ZERO,
775            depth,
776            None,
777        )
778    }
779
780    /// Test-only root constructor that accepts an explicit
781    /// [`CheckpointCoalescer`], letting tests force small [`BatchLimits`]
782    /// so batch splitting is observable with a handful of updates.
783    #[cfg(test)]
784    pub(crate) fn new_root_with_client_and_coalescer(
785        execution_arn: String,
786        lambda_context: lambda_runtime::Context,
787        checkpoint_log: Arc<CheckpointLog>,
788        client: Arc<dyn ExecutionClient>,
789        checkpoint_token: String,
790        coalescer: CheckpointCoalescer,
791    ) -> Self {
792        let engine = Arc::new(EngineState::new_root(checkpoint_log));
793        let replay_span = crate::tracing_layer::execution_span(
794            &execution_arn,
795            &lambda_context.request_id,
796            lambda_context.tenant_id.as_deref(),
797            engine.is_replaying(),
798        );
799        Self {
800            inner: Arc::new(Inner {
801                execution_arn,
802                lambda_context,
803                engine,
804                coordinator: InvocationCoordinator::new(),
805                task_ownership: Arc::new(TaskOwnership::new_current()),
806                execution_client: Some(client),
807                checkpoint_token: Arc::new(Mutex::new(checkpoint_token)),
808                coalescer: Arc::new(coalescer),
809                #[cfg(test)]
810                hooks: Arc::default(),
811                parent_wire_id: None,
812                admission: Arc::new(Admission::new()),
813                owners: Arc::from(Vec::new()),
814                replay_span,
815                plugins: None,
816                nesting_level: 0,
817                preserve_child_depth: 0,
818                rebuilding: false,
819            }),
820            awaited: None,
821        }
822    }
823    /// Creates the child context of the child operation at
824    /// `parent_positional_id`. `name` is that operation's name; the child's
825    /// span carries it as `operationName`, with the operation's wire ID as
826    /// `operationId`.
827    pub(crate) fn new_child(&self, parent_positional_id: &str, name: Option<&str>) -> Self {
828        self.new_child_rebuilding(parent_positional_id, name, false)
829    }
830
831    /// [`Self::new_child`] for a body that may run again to rebuild the
832    /// result of a recorded success (`ReplayChildren`). With `rebuilding`
833    /// true, or when this context is rebuilding, the child and every
834    /// context derived from it are rebuilding: an operation there returns
835    /// only its recorded outcome (see [`crate::start::rebuild_holds`]).
836    pub(crate) fn new_child_rebuilding(
837        &self,
838        parent_positional_id: &str,
839        name: Option<&str>,
840        rebuilding: bool,
841    ) -> Self {
842        let wire_id = crate::engine::compute_wire_id_public(parent_positional_id);
843        let engine = Arc::new(EngineState::new_child(
844            parent_positional_id,
845            Arc::clone(&self.inner.engine.checkpoint_log),
846        ));
847        // The child namespace has its own replay high-water mark: nested
848        // operations can still be replaying while the parent is already
849        // live. Give it its own span, initialized from the CHILD engine's
850        // replay status; the child's mints keep it current, and the parent
851        // span keeps the value the parent's own mints gave it. The span is
852        // a child of the span current here, so a span a plugin entered
853        // around the code that creates this context is an ancestor of the
854        // child body's records. The replay filter decides by this span and
855        // the SDK operation spans inside this namespace, never by a parent
856        // namespace's span.
857        let replay_span = crate::tracing_layer::scoped_execution_span(
858            &self.inner.execution_arn,
859            &self.inner.lambda_context.request_id,
860            self.inner.lambda_context.tenant_id.as_deref(),
861            &wire_id,
862            name,
863            engine.is_replaying(),
864        );
865        Self {
866            inner: Arc::new(Inner {
867                execution_arn: self.inner.execution_arn.clone(),
868                lambda_context: self.inner.lambda_context.clone(),
869                engine,
870                coordinator: Arc::clone(&self.inner.coordinator),
871                task_ownership: Arc::clone(&self.inner.task_ownership),
872                execution_client: self.inner.execution_client.clone(),
873                checkpoint_token: Arc::clone(&self.inner.checkpoint_token),
874                coalescer: Arc::clone(&self.inner.coalescer),
875                #[cfg(test)]
876                hooks: Arc::clone(&self.inner.hooks),
877                parent_wire_id: Some(wire_id),
878                admission: Arc::clone(&self.inner.admission),
879                owners: self.owners_with(parent_positional_id),
880                replay_span,
881                plugins: self.inner.plugins.clone(),
882                nesting_level: self.inner.nesting_level.saturating_add(1),
883                preserve_child_depth: self.inner.preserve_child_depth,
884                rebuilding: self.inner.rebuilding || rebuilding,
885            }),
886            awaited: None,
887        }
888    }
889
890    /// Creates the context of the virtual child context at
891    /// `child_positional_id`, which this context claimed. It equals
892    /// [`Self::new_child`], except that its operations name this context's
893    /// parent as their parent, it keeps this context's nesting level, and
894    /// it runs in this context's owners: a virtual child writes no record,
895    /// so it is not an owner, and its scope never closes on its own. `name`
896    /// is the virtual operation's name; the child's span carries it as
897    /// `operationName`, with the virtual operation's wire ID as
898    /// `operationId`.
899    pub(crate) fn new_virtual_child(&self, child_positional_id: &str, name: Option<&str>) -> Self {
900        let wire_id = crate::engine::compute_wire_id_public(child_positional_id);
901        let engine = Arc::new(EngineState::new_child(
902            child_positional_id,
903            Arc::clone(&self.inner.engine.checkpoint_log),
904        ));
905        // The virtual namespace has its own replay high-water mark, so it
906        // gets its own span, a child of the current span, as a child from
907        // `new_child` does. It carries the virtual operation's stable ID,
908        // and no record is written for it.
909        let replay_span = crate::tracing_layer::scoped_execution_span(
910            &self.inner.execution_arn,
911            &self.inner.lambda_context.request_id,
912            self.inner.lambda_context.tenant_id.as_deref(),
913            &wire_id,
914            name,
915            engine.is_replaying(),
916        );
917        Self {
918            inner: Arc::new(Inner {
919                execution_arn: self.inner.execution_arn.clone(),
920                lambda_context: self.inner.lambda_context.clone(),
921                engine,
922                coordinator: Arc::clone(&self.inner.coordinator),
923                task_ownership: Arc::clone(&self.inner.task_ownership),
924                execution_client: self.inner.execution_client.clone(),
925                checkpoint_token: Arc::clone(&self.inner.checkpoint_token),
926                coalescer: Arc::clone(&self.inner.coalescer),
927                #[cfg(test)]
928                hooks: Arc::clone(&self.inner.hooks),
929                parent_wire_id: self.inner.parent_wire_id.clone(),
930                admission: Arc::clone(&self.inner.admission),
931                owners: Arc::clone(&self.inner.owners),
932                replay_span,
933                plugins: self.inner.plugins.clone(),
934                nesting_level: self.inner.nesting_level,
935                preserve_child_depth: self.inner.preserve_child_depth,
936                rebuilding: self.inner.rebuilding,
937            }),
938            awaited: None,
939        }
940    }
941
942    /// Fails the virtual child context claimed at `op_id` when the log
943    /// holds a record at its ID: a virtual child writes no record, so a
944    /// record there was written by another operation, such as a non-virtual
945    /// child that the handler later switched to virtual. The error is a
946    /// `NonDeterministicExecution` mismatch, recorded as the invocation's
947    /// fatal error, whose claimed identity is `virtual Context` with
948    /// `sub_type` and `name`. Without a record it returns `Ok(())`.
949    pub(crate) fn reject_record_at_virtual_child(
950        &self,
951        op_id: &OperationId,
952        sub_type: &str,
953        name: Option<&str>,
954    ) -> Result<(), OperationError> {
955        let Some(expected) = self.with_checkpoint_record(op_id.positional(), |record| {
956            format_op_identity(
957                record.op_type.as_deref().unwrap_or("Unknown"),
958                record.sub_type.as_deref(),
959                record.op_name.as_deref(),
960            )
961        }) else {
962            return Ok(());
963        };
964        Err(self.replay_mismatch_error(
965            op_id.wire(),
966            &expected,
967            "virtual Context",
968            Some(sub_type),
969            name,
970        ))
971    }
972
973    /// Whether a context for the child namespace `child_positional_id`
974    /// would replay before its first claim: the first operation of that
975    /// namespace has a checkpoint record. It is the
976    /// [`Self::is_replaying`] a child context created now starts with.
977    pub(crate) fn namespace_is_replaying(&self, child_positional_id: &str) -> bool {
978        EngineState::new_child(
979            child_positional_id,
980            Arc::clone(&self.inner.engine.checkpoint_log),
981        )
982        .is_replaying()
983    }
984
985    /// Whether a context-backed record created on this context at `levels_below`
986    /// levels under it records `ReplayChildren` on its terminal update.
987    pub(crate) fn preserves_children(&self, levels_below: u32) -> bool {
988        let level = self.inner.nesting_level.saturating_add(levels_below);
989        level <= self.inner.preserve_child_depth
990    }
991
992    /// Whether this context runs inside a rebuilt body (see
993    /// [`Self::new_child_rebuilding`]).
994    pub(crate) fn is_rebuilding(&self) -> bool {
995        self.inner.rebuilding
996    }
997
998    /// Mints the next operation ID (internal engine concern).
999    ///
1000    /// Also re-records THIS namespace's span `isReplay` flag: after each
1001    /// claim, the namespace is replaying iff the NEXT operation to be
1002    /// claimed in it has a checkpoint record. This keeps replay-aware log
1003    /// filters (see `tracing_layer::ReplayFilterLayer`) in step with the
1004    /// live replay status as each namespace crosses its own replay
1005    /// high-water mark.
1006    pub(crate) fn mint_id(&self) -> OperationId {
1007        let id = self.inner.engine.mint_id();
1008        self.inner.replay_span.record(
1009            crate::tracing_layer::fields::IS_REPLAY,
1010            self.inner.engine.is_replaying(),
1011        );
1012        id
1013    }
1014
1015    /// Returns this context's positional prefix: `""` for the root context,
1016    /// the positional ID of the operation that created it for a child.
1017    pub(crate) fn prefix(&self) -> &str {
1018        self.inner.engine.id_counter.prefix()
1019    }
1020
1021    /// Claims the next operation ID for the public operation `method`, and
1022    /// checks that the caller may claim it on this context.
1023    ///
1024    /// The SDK records which body it is polling in [`ACTIVE_BODY`]. The
1025    /// check compares that body with this context. The outcomes are:
1026    ///
1027    /// - No body is active, or the body of this context is active: the
1028    ///   caller must own the context (see [`TaskOwnership`]). A task or
1029    ///   thread that does not own it fails the execution with a context
1030    ///   usage error.
1031    /// - The body of another context is active: the execution fails with a
1032    ///   context usage error that names both contexts.
1033    /// - A step body or a `wait_for_condition` check is active: the
1034    ///   execution fails with a context usage error that names the
1035    ///   operation.
1036    /// - Plugin code is active: the SDK records a context usage error.
1037    ///
1038    /// The failure is recorded as an execution-fatal error, which closes
1039    /// checkpoint admission: no update requested after the record is
1040    /// written, so an operation whose next update is refused keeps the last
1041    /// state written. When it is recorded before the driver's arbitration,
1042    /// the invocation answers `FAILED` on the first invocation. Only plugin code can record it
1043    /// after the arbitration, either before `drive_invocation` returns or
1044    /// after; such a record changes neither the response nor the
1045    /// late-panic policy. Without the
1046    /// check, the misuse surfaces one invocation later as a
1047    /// non-determinism error that does not name the cause. The ID is
1048    /// claimed in every case, so the caller builds its builder as usual.
1049    ///
1050    /// Only the public operation methods call this. The SDK's own internal
1051    /// claims (inside `wait_for_callback`) call [`Self::mint_id`], because
1052    /// they run on the composite's context while a child body may be
1053    /// active.
1054    ///
1055    /// [`TaskOwnership`]: crate::driver::TaskOwnership
1056    pub(crate) fn claim_operation_id(&self, method: &'static str) -> OperationId {
1057        let message = ACTIVE_BODY
1058            .try_with(|active| match active {
1059                ActiveBody::Context(prefix) if prefix == self.prefix() => None,
1060                ActiveBody::Context(prefix) => Some(format!(
1061                    "context usage error in \"{method}\": the operation was called on context \
1062                     \"{calling}\", but the SDK is running the body of context \"{active}\". \
1063                     Inside run_in_child_context, with_retry, map and parallel, call operations \
1064                     on the context passed to the closure.",
1065                    calling = display_prefix(self.prefix()),
1066                    active = display_prefix(prefix),
1067                )),
1068                ActiveBody::Step(id) => Some(format!(
1069                    "context usage error in \"{method}\": the operation was called inside the \
1070                     body of operation \"{id}\". A step body or a wait_for_condition check \
1071                     cannot call durable operations; use run_in_child_context to group \
1072                     operations."
1073                )),
1074                ActiveBody::Plugin => Some(format!(
1075                    "context usage error in \"{method}\": the operation was called inside \
1076                     plugin code. Plugin code cannot call durable operations."
1077                )),
1078                ActiveBody::Outcome(id) => Some(format!(
1079                    "context usage error in \"{method}\": the operation was called while the \
1080                     SDK records the outcome of operation \"{id}\". An error's Display and \
1081                     source cannot call durable operations."
1082                )),
1083                ActiveBody::Unset => None,
1084            })
1085            .ok()
1086            .flatten();
1087        let message = message.or_else(|| {
1088            self.inner
1089                .task_ownership
1090                .check_current_task()
1091                .err()
1092                .map(|_| foreign_caller_message(method))
1093        });
1094        if let Some(message) = message {
1095            self.coordinator().record_fatal("Error".to_owned(), message);
1096        }
1097        self.mint_id()
1098    }
1099
1100    /// Returns this namespace's `durable_execution` span. The root task (for
1101    /// a root context) or the child body's runner (for a child context)
1102    /// enters it outside the plugin frames and wraps, so log events inherit
1103    /// the execution ARN, request ID, and the namespace's live `isReplay`
1104    /// flag, and a span a plugin enters around `next()` is its child.
1105    pub(crate) fn replay_span(&self) -> tracing::Span {
1106        self.inner.replay_span.clone()
1107    }
1108
1109    /// Emits the
1110    /// [`operation_replayed`](crate::observability::event_names::OPERATION_REPLAYED)
1111    /// lifecycle event: the operation's recorded terminal outcome is being
1112    /// returned without re-running it (see [`crate::observability`]).
1113    ///
1114    /// `recorded_attempt` is the checkpoint record's retry count; the event
1115    /// reports the 1-based attempt that produced the recorded outcome.
1116    pub(crate) fn emit_operation_replayed(
1117        &self,
1118        wire_id: &str,
1119        operation_name: Option<&str>,
1120        operation_type: &str,
1121        operation_sub_type: Option<&str>,
1122        recorded_attempt: u32,
1123    ) {
1124        crate::tracing_layer::operation_replayed_event(&crate::tracing_layer::OperationIdentity {
1125            execution_arn: self.execution_arn(),
1126            request_id: &self.lambda_context().request_id,
1127            operation_id: wire_id,
1128            operation_name,
1129            operation_type,
1130            operation_sub_type,
1131            attempt: recorded_attempt.saturating_add(1),
1132        });
1133    }
1134
1135    /// FLAT-nesting counterpart of [`Self::new_child`]: a child that
1136    /// reports `parent_wire_id_override` as its operations' parent (the
1137    /// batch parent, not the virtual child). The child's span carries the
1138    /// virtual child's wire ID as `operationId` and `name` as
1139    /// `operationName`.
1140    pub(crate) fn new_flat_child(
1141        &self,
1142        child_positional_id: &str,
1143        parent_wire_id_override: &str,
1144        name: Option<&str>,
1145    ) -> Self {
1146        let engine = Arc::new(EngineState::new_child(
1147            child_positional_id,
1148            Arc::clone(&self.inner.engine.checkpoint_log),
1149        ));
1150        // A flat branch has its own namespace and replay high-water mark,
1151        // so it gets its own span, a child of the current span, as a child
1152        // from `new_child` does. It carries the virtual child's stable ID,
1153        // and no record is written for it.
1154        let replay_span = crate::tracing_layer::scoped_execution_span(
1155            &self.inner.execution_arn,
1156            &self.inner.lambda_context.request_id,
1157            self.inner.lambda_context.tenant_id.as_deref(),
1158            &crate::engine::compute_wire_id_public(child_positional_id),
1159            name,
1160            engine.is_replaying(),
1161        );
1162        Self {
1163            inner: Arc::new(Inner {
1164                execution_arn: self.inner.execution_arn.clone(),
1165                lambda_context: self.inner.lambda_context.clone(),
1166                engine,
1167                coordinator: Arc::clone(&self.inner.coordinator),
1168                task_ownership: Arc::clone(&self.inner.task_ownership),
1169                execution_client: self.inner.execution_client.clone(),
1170                checkpoint_token: Arc::clone(&self.inner.checkpoint_token),
1171                coalescer: Arc::clone(&self.inner.coalescer),
1172                #[cfg(test)]
1173                hooks: Arc::clone(&self.inner.hooks),
1174                parent_wire_id: Some(parent_wire_id_override.to_owned()),
1175                admission: Arc::clone(&self.inner.admission),
1176                owners: self.owners_with(child_positional_id),
1177                replay_span,
1178                plugins: self.inner.plugins.clone(),
1179                nesting_level: self.inner.nesting_level,
1180                preserve_child_depth: self.inner.preserve_child_depth,
1181                rebuilding: self.inner.rebuilding,
1182            }),
1183            awaited: None,
1184        }
1185    }
1186
1187    /// The scopes of this invocation: which are closed, and the tasks
1188    /// started in each (see [`Admission`]).
1189    pub(crate) fn admission(&self) -> &Arc<Admission> {
1190        &self.inner.admission
1191    }
1192
1193    /// The owners this context runs in, outermost first. Empty at the root.
1194    pub(crate) fn owners(&self) -> &[String] {
1195        &self.inner.owners
1196    }
1197
1198    /// Returns at once when no owner of this context is abandoned, and
1199    /// never returns otherwise (see [`Admission`]). Every operation
1200    /// terminal awaits it before it runs the operation, so an abandoned
1201    /// owner starts no operation and replays no recorded one.
1202    pub(crate) async fn admit_operation(&self) {
1203        if !self.admission().admits(self.owners()) {
1204            std::future::pending::<()>().await;
1205        }
1206    }
1207
1208    /// This context's owners followed by the wire ID of `positional_id`:
1209    /// the owners of a child context created for that operation.
1210    fn owners_with(&self, positional_id: &str) -> Arc<[String]> {
1211        self.inner
1212            .owners
1213            .iter()
1214            .cloned()
1215            .chain(std::iter::once(crate::engine::compute_wire_id_public(
1216                positional_id,
1217            )))
1218            .collect()
1219    }
1220
1221    /// Installs the checkpoint writer's pre-lock gate (see [`PreLockGate`]).
1222    /// The root context and every child share it.
1223    #[cfg(test)]
1224    pub(crate) fn set_pre_lock_gate(&self, gate: PreLockGate) {
1225        self.hooks().gate = Some(gate);
1226    }
1227
1228    /// Installs the submission recorder (see [`SubmissionRecorder`]). The
1229    /// root context and every child share it.
1230    #[cfg(test)]
1231    pub(crate) fn set_submission_recorder(&self, recorder: SubmissionRecorder) {
1232        self.hooks().submissions = Some(recorder);
1233    }
1234
1235    /// Installs the settler-entry recorder (see [`SettlerEntryRecorder`]).
1236    /// The root context and every child share it.
1237    #[cfg(test)]
1238    pub(crate) fn set_settler_entry_recorder(&self, recorder: SettlerEntryRecorder) {
1239        self.hooks().settler_entries = Some(recorder);
1240    }
1241
1242    /// The test hooks shared by every context of the invocation.
1243    #[cfg(test)]
1244    fn hooks(&self) -> std::sync::MutexGuard<'_, TestHooks> {
1245        self.inner
1246            .hooks
1247            .lock()
1248            .unwrap_or_else(std::sync::PoisonError::into_inner)
1249    }
1250
1251    /// Waits on the pre-lock gate for `updates`, when one is installed and
1252    /// returns a `Notify` for them.
1253    #[cfg(test)]
1254    async fn pass_gate(&self, updates: &[OperationUpdate]) {
1255        let gate = self.hooks().gate.clone();
1256        if let Some(notify) = gate.and_then(|gate| gate(updates)) {
1257            notify.notified().await;
1258        }
1259    }
1260
1261    /// Passes `updates` and `write`'s first-poll flag to the submission
1262    /// recorder, when one is installed.
1263    #[cfg(test)]
1264    fn record_submission(&self, updates: &[OperationUpdate], write: &SubmittedWrite) {
1265        let recorder = self.hooks().submissions.clone();
1266        if let Some(recorder) = recorder
1267            && !updates.is_empty()
1268        {
1269            recorder(updates, &write.first_poll);
1270        }
1271    }
1272
1273    /// Waits for `start`, the retained `START` write of the child-backed
1274    /// operation `wire_id`, before the operation writes its outcome
1275    /// `action`. A failed `START` ends the invocation, as
1276    /// [`Self::checkpoint_failure_unrecoverable`] describes, and the
1277    /// outcome is never written. `None` means the operation's `START` was
1278    /// recorded before this cycle.
1279    pub(crate) async fn await_retained_start(
1280        &self,
1281        wire_id: &str,
1282        start: Option<SubmittedWrite>,
1283        action: OperationAction,
1284    ) -> Result<(), OperationError> {
1285        #[cfg(test)]
1286        {
1287            let recorder = self.hooks().settler_entries.clone();
1288            if let Some(recorder) = recorder {
1289                recorder(wire_id, action.clone());
1290            }
1291        }
1292        #[cfg(not(test))]
1293        let _ = (wire_id, action);
1294        if let Some(start) = start
1295            && let Err(err) = start.await
1296        {
1297            return self.checkpoint_failure_unrecoverable(err).await;
1298        }
1299        Ok(())
1300    }
1301
1302    /// Buffers `updates` in the coalescer's open batch without starting a
1303    /// flusher, so only a flush point writes them.
1304    #[cfg(test)]
1305    pub(crate) fn buffer_unflushed(&self, updates: Vec<OperationUpdate>) {
1306        let tracked = self.track_updates(updates);
1307        self.inner
1308            .coalescer
1309            .join(self.admission(), self.coordinator(), self.owners(), tracked)
1310            .expect("the join is admitted");
1311    }
1312
1313    /// The IDs of the updates that wait to be sent: those of the
1314    /// coalescer's open batch.
1315    #[cfg(test)]
1316    pub(crate) fn queued_update_ids(&self) -> Vec<String> {
1317        self.inner.coalescer.pending_ids()
1318    }
1319
1320    /// The checkpoint token mutex, which a checkpoint call holds.
1321    #[cfg(test)]
1322    pub(crate) fn checkpoint_token_mutex(&self) -> Arc<Mutex<String>> {
1323        Arc::clone(&self.inner.checkpoint_token)
1324    }
1325
1326    /// The plugins of the invocation, shared by every context derived from
1327    /// the root. `None` without a registered plugin factory.
1328    pub(crate) fn plugins(&self) -> Option<&crate::plugin::InvocationPlugins> {
1329        self.inner.plugins.as_deref()
1330    }
1331
1332    /// Delivers the queued change sets of the plugins to
1333    /// `on_operation_change`, and waits until every change set queued
1334    /// before the call was delivered, also when another task delivers it
1335    /// (see
1336    /// [`InvocationPlugins::changes_delivered`](crate::plugin::InvocationPlugins::changes_delivered)).
1337    /// So an operation that awaits this before it fires
1338    /// `on_operation_start` or `on_operation_end` fires that hook after its
1339    /// change. Does nothing without plugins. The caller must hold no
1340    /// checkpoint lock.
1341    pub(crate) async fn report_operation_changes(&self) {
1342        if let Some(plugins) = self.plugins() {
1343            plugins
1344                .changes_delivered(&self.inner.engine.checkpoint_log)
1345                .await;
1346        }
1347    }
1348
1349    /// [`Self::plugins`], shared: a user-function runner's own frame node
1350    /// holds them.
1351    pub(crate) fn shared_plugins(&self) -> Option<Arc<crate::plugin::InvocationPlugins>> {
1352        self.inner.plugins.clone()
1353    }
1354
1355    /// Returns the invocation coordinator, shared by every context of the
1356    /// invocation.
1357    pub(crate) fn coordinator(&self) -> &Arc<InvocationCoordinator> {
1358        &self.inner.coordinator
1359    }
1360
1361    /// Awaits `fut` while holding an activity permit, so the invocation
1362    /// does not suspend while `fut` runs. When the coordinator refuses the
1363    /// permit (the phase left `Running`), never resolves.
1364    ///
1365    /// Wraps the step body, the `wait_for_condition` check and every serdes
1366    /// call: work that can make progress without the SDK polling it.
1367    pub(crate) async fn hold_activity<F: Future>(&self, fut: F) -> F::Output {
1368        let Some(permit) = self.inner.coordinator.acquire(PermitKind::Activity) else {
1369            return std::future::pending().await;
1370        };
1371        let output = fut.await;
1372        drop(permit);
1373        output
1374    }
1375
1376    /// Resolves when the record of `wire_id` satisfies `ready`.
1377    ///
1378    /// Registers a wait on its first poll, then reads the record. A missing
1379    /// record is not ready. When the record is ready, the registration is
1380    /// dropped and the future resolves. Otherwise it returns `Pending`; a
1381    /// merged checkpoint response that carries the operation wakes it, and
1382    /// each later poll updates the waker and reads the record again.
1383    ///
1384    /// Registering before the read means a merge between the read and the
1385    /// end of the poll still wakes the waiting task. When the coordinator
1386    /// refuses the registration (the phase left `Running`), never resolves.
1387    /// The status is read with
1388    /// [`CheckpointLog::published_status_view`], so a status that a
1389    /// checkpoint response changed is read only after the response's change
1390    /// set was queued, also on another worker.
1391    pub(crate) async fn wait_for_record(
1392        &self,
1393        wire_id: &str,
1394        ready: fn(&CheckpointStatus) -> bool,
1395    ) {
1396        let mut registration: Option<crate::coordinator::WaitRegistration> = None;
1397        std::future::poll_fn(|cx| {
1398            if let Some(live) = &registration {
1399                live.update_waker(cx.waker());
1400            } else {
1401                // The refresh schedule starts from the record's end time
1402                // and status. A missing record is taken as `STARTED` with
1403                // no end time.
1404                let (status, due_at) = self
1405                    .refresh_view(wire_id)
1406                    .unwrap_or((CheckpointStatus::Started, None));
1407                match self.inner.coordinator.register_wait(
1408                    wire_id,
1409                    due_at,
1410                    &status,
1411                    self.awaited.clone(),
1412                    cx.waker(),
1413                ) {
1414                    Some(live) => registration = Some(live),
1415                    None => return std::task::Poll::Pending,
1416                }
1417            }
1418            let is_ready = self
1419                .inner
1420                .engine
1421                .checkpoint_log
1422                .published_status_view(wire_id)
1423                .is_some_and(|view| ready(&view.status));
1424            if is_ready {
1425                registration = None;
1426                std::task::Poll::Ready(())
1427            } else {
1428                if let Some(live) = &registration {
1429                    live.resume_schedule();
1430                }
1431                std::task::Poll::Pending
1432            }
1433        })
1434        .await;
1435        // The response that readied the record queued its change set. It is
1436        // delivered before the operation reads the record again, so it
1437        // comes before the operation's end hook.
1438        self.report_operation_changes().await;
1439    }
1440
1441    /// The status and end time of the record of `wire_id`, as the refresh
1442    /// scheduler reads them.
1443    pub(crate) fn refresh_view(
1444        &self,
1445        wire_id: &str,
1446    ) -> Option<(CheckpointStatus, Option<std::time::SystemTime>)> {
1447        self.inner
1448            .engine
1449            .checkpoint_log
1450            .with_record(wire_id, |record| (record.status.clone(), record.due_at))
1451    }
1452
1453    /// A clone of this context whose wait registrations carry `flag`, the
1454    /// first-poll flag of the handle of an operation started on its own
1455    /// task. The refresh scheduler skips those registrations until the
1456    /// handle is first polled and sets the flag.
1457    pub(crate) fn with_awaited(&self, flag: Arc<std::sync::atomic::AtomicBool>) -> Self {
1458        Self {
1459            inner: Arc::clone(&self.inner),
1460            awaited: Some(flag),
1461        }
1462    }
1463
1464    /// Waits until the record of the operation at `positional_id` satisfies
1465    /// `ready` (see [`Self::wait_for_record`]), then returns its view.
1466    pub(crate) async fn wait_for_record_view(
1467        &self,
1468        positional_id: &str,
1469        ready: fn(&CheckpointStatus) -> bool,
1470    ) -> CheckpointStatusView {
1471        let wire_id = crate::engine::compute_wire_id_public(positional_id);
1472        self.wait_for_record(&wire_id, ready).await;
1473        #[expect(clippy::expect_used)]
1474        // reason: `wait_for_record` resolves only once the record exists,
1475        // and the checkpoint log never removes a record during an
1476        // invocation.
1477        self.checkpoint_status_view(positional_id)
1478            .expect("a record that satisfied the wait is still in the log")
1479    }
1480
1481    /// Returns the time left until the Lambda deadline: `None` when the
1482    /// deadline is 0 (unknown), otherwise the deadline (milliseconds since
1483    /// the Unix epoch) minus now, and `Duration::ZERO` once it passed.
1484    pub(crate) fn remaining_time(&self) -> Option<Duration> {
1485        remaining_until(
1486            self.inner.lambda_context.deadline,
1487            std::time::SystemTime::now(),
1488        )
1489    }
1490
1491    /// Returns a reference to the task-ownership tracker.
1492    pub(crate) fn task_ownership(&self) -> &Arc<TaskOwnership> {
1493        &self.inner.task_ownership
1494    }
1495
1496    /// Checks task ownership and returns an `OperationError` if the caller
1497    /// is not authorized. Used by every durable operation entry point.
1498    pub(crate) fn enforce_task_ownership(&self) -> Result<(), OperationError> {
1499        self.inner
1500            .task_ownership
1501            .check_current_task()
1502            .map_err(|msg| {
1503                self.coordinator()
1504                    .record_fatal("Error".to_owned(), foreign_caller_message("execute"));
1505                OperationError::from_kind(OperationErrorKind::Step(StepError::new(
1506                    StepErrorKind::ExecutionFailed,
1507                    Some(msg.into()),
1508                )))
1509            })
1510    }
1511
1512    /// Reads the checkpoint record for the given positional ID in place,
1513    /// applying `f` under the log's read guard.
1514    ///
1515    /// Returns `None` when no record exists. This (and the targeted
1516    /// accessors below) replaces the removed full-record clone: `f` borrows
1517    /// the stored record, so only what `f` returns is cloned. `f` must not
1518    /// touch the checkpoint log (the read guard is held while it runs).
1519    ///
1520    /// NOTE: The checkpoint log is keyed by wire ID (the hash), which is
1521    /// what the backend returns in Operations[].Id. We look up by wire ID
1522    /// computed from the positional ID.
1523    pub(crate) fn with_checkpoint_record<R>(
1524        &self,
1525        positional_id: &str,
1526        f: impl FnOnce(&CheckpointRecord) -> R,
1527    ) -> Option<R> {
1528        let wire_id = crate::engine::compute_wire_id_public(positional_id);
1529        self.inner.engine.checkpoint_log.with_record(&wire_id, f)
1530    }
1531
1532    /// [`Self::with_checkpoint_record`], for the record stored under the
1533    /// wire ID `wire_id`.
1534    pub(crate) fn with_checkpoint_record_by_wire<R>(
1535        &self,
1536        wire_id: &str,
1537        f: impl FnOnce(&CheckpointRecord) -> R,
1538    ) -> Option<R> {
1539        self.inner.engine.checkpoint_log.with_record(wire_id, f)
1540    }
1541
1542    /// Returns the compact view (status, attempt,
1543    /// `replay_children`) of the checkpoint record for the given positional
1544    /// ID, without cloning any of the record's owned strings.
1545    pub(crate) fn checkpoint_status_view(
1546        &self,
1547        positional_id: &str,
1548    ) -> Option<CheckpointStatusView> {
1549        let wire_id = crate::engine::compute_wire_id_public(positional_id);
1550        self.inner.engine.checkpoint_log.status_view(&wire_id)
1551    }
1552
1553    /// Returns whether a checkpoint record exists for the given positional
1554    /// ID, without cloning anything.
1555    ///
1556    /// Production operation paths read record presence through
1557    /// [`Self::checkpoint_view_validated`] (which also validates identity);
1558    /// this direct form is retained for unit tests.
1559    #[cfg(test)]
1560    pub(crate) fn has_checkpoint_record(&self, positional_id: &str) -> bool {
1561        let wire_id = crate::engine::compute_wire_id_public(positional_id);
1562        self.inner.engine.checkpoint_log.contains(&wire_id)
1563    }
1564
1565    /// Returns the recorded result payload for the given positional ID,
1566    /// cloning only that one string.
1567    ///
1568    /// `None` means either no record exists or the record carries no
1569    /// result: callers that need to distinguish should use
1570    /// [`Self::with_checkpoint_record`].
1571    pub(crate) fn checkpoint_result_payload(&self, positional_id: &str) -> Option<String> {
1572        self.with_checkpoint_record(positional_id, |record| record.result.clone())
1573            .flatten()
1574    }
1575
1576    /// Returns the backend-assigned callback ID for the given positional
1577    /// ID, cloning only that one string.
1578    pub(crate) fn checkpoint_callback_id(&self, positional_id: &str) -> Option<String> {
1579        self.with_checkpoint_record(positional_id, |record| record.callback_id.clone())
1580            .flatten()
1581    }
1582
1583    /// Returns the recorded wire failure record for the given positional
1584    /// ID, cloning only the failure strings. `None` means no record
1585    /// exists.
1586    pub(crate) fn checkpoint_wire_error(
1587        &self,
1588        positional_id: &str,
1589    ) -> Option<crate::error::WireError> {
1590        self.with_checkpoint_record(positional_id, |record| {
1591            crate::error::WireError::new(record.error_type.clone(), record.error_message.clone())
1592                .with_error_data(record.error_data.clone())
1593                .with_stack_trace(record.stack_trace.clone().unwrap_or_default())
1594        })
1595    }
1596
1597    /// Returns the terminal-replay projection (status, `replay_children`,
1598    /// and the result/error strings) of the checkpoint record for the
1599    /// given positional ID, cloning only those payload strings.
1600    ///
1601    /// This is what the map/parallel replay helpers consume when
1602    /// reconstructing a terminal batch or child item; the record's invoke,
1603    /// callback, attempt, ID, and identity fields are never cloned. `None`
1604    /// means no record exists.
1605    pub(crate) fn checkpoint_terminal_replay(
1606        &self,
1607        positional_id: &str,
1608    ) -> Option<TerminalReplaySnapshot> {
1609        self.with_checkpoint_record(positional_id, |record| TerminalReplaySnapshot {
1610            status: record.status.clone(),
1611            replay_children: record.replay_children,
1612            result: record.result.clone(),
1613            error_message: record.error_message.clone(),
1614            error_type: record.error_type.clone(),
1615            error_data: record.error_data.clone(),
1616            stack_trace: record.stack_trace.clone(),
1617        })
1618    }
1619
1620    /// Validates the claimed operation identity against the checkpoint
1621    /// record and returns the compact status view, in a single read-guard
1622    /// pass: the common preamble of every operation's replay check.
1623    ///
1624    /// Returns `Ok(None)` when no record exists (live position, nothing to
1625    /// validate), `Ok(Some(view))` when the identity matches, and the
1626    /// `NonDeterministicExecution` error (also recorded on the fatal slot,
1627    /// exactly as [`Self::validate_replay_identity`] does) on mismatch,
1628    /// or when the record carries a [`CheckpointStatus::Unknown`] status
1629    /// this SDK version cannot interpret, in which case the error names
1630    /// the raw status. Nothing is cloned on the match path beyond the
1631    /// status itself.
1632    pub(crate) fn checkpoint_view_validated(
1633        &self,
1634        positional_id: &str,
1635        wire_id: &str,
1636        claimed_type: &str,
1637        claimed_sub_type: Option<&str>,
1638        claimed_name: Option<&str>,
1639    ) -> Result<Option<CheckpointStatusView>, OperationError> {
1640        let Some((mismatch, view)) = self.with_checkpoint_record(positional_id, |record| {
1641            (
1642                replay_identity_mismatch(record, claimed_type, claimed_sub_type, claimed_name),
1643                CheckpointStatusView {
1644                    status: record.status.clone(),
1645                    attempt: record.attempt,
1646                    replay_children: record.replay_children,
1647                },
1648            )
1649        }) else {
1650            return Ok(None);
1651        };
1652        if let Some(expected) = mismatch {
1653            // Built (and the fatal slot written) OUTSIDE the read guard.
1654            return Err(self.replay_mismatch_error(
1655                wire_id,
1656                &expected,
1657                claimed_type,
1658                claimed_sub_type,
1659                claimed_name,
1660            ));
1661        }
1662        if let CheckpointStatus::Unknown(raw) = &view.status {
1663            // A status this SDK cannot interpret must never be acted on:
1664            // guessing non-terminal re-runs completed work, and guessing
1665            // terminal fabricates an outcome that was never recorded. Fail
1666            // the execution naming the raw status (issue #45).
1667            return Err(self.unrecognized_status_error(wire_id, raw));
1668        }
1669        Ok(Some(view))
1670    }
1671
1672    /// Eagerly validates a claimed operation's replay identity against the
1673    /// checkpoint log, BEFORE the operation future first runs.
1674    ///
1675    /// Builders call this when they are finalized into a
1676    /// [`crate::DurableFuture`] (`.future()`, `.spawn()`, or `.await` via
1677    /// `IntoFuture`). Validating at finalization instead of first poll makes
1678    /// fatal propagation scheduler-independent: a short-circuiting combinator
1679    /// (`select_ok`, `race`, `try_join_all`) aborts losers as soon as a
1680    /// winner settles, so a mismatching constituent might otherwise never be
1681    /// polled and the mismatch never recorded. The fatal slot is written
1682    /// synchronously here (inside [`Self::validate_replay_identity`]), so the
1683    /// invocation driver fails the execution with the dedicated error no
1684    /// matter which sibling settles first.
1685    ///
1686    /// A position with no checkpoint record is live (nothing to validate),
1687    /// and a matching identity is a no-op: an unchanged handler behaves
1688    /// identically.
1689    pub(crate) fn preflight_replay_identity(
1690        &self,
1691        op_id: &OperationId,
1692        claimed_type: &str,
1693        claimed_sub_type: Option<&str>,
1694        claimed_name: Option<&str>,
1695    ) -> Result<(), OperationError> {
1696        // An abandoned owner replays no operation, so its claim is not
1697        // checked against history. A mismatch here would record a fatal
1698        // error and resolve the future. Skipping the check leaves the
1699        // future to the admission check in `DurableFuture::for_operation`,
1700        // which never resolves it. An owner abandoned after this check is
1701        // stopped there too.
1702        if !self.admission().admits(self.owners()) {
1703            return Ok(());
1704        }
1705        self.checkpoint_view_validated(
1706            op_id.positional(),
1707            op_id.wire(),
1708            claimed_type,
1709            claimed_sub_type,
1710            claimed_name,
1711        )?;
1712        Ok(())
1713    }
1714
1715    /// Validates that the claimed operation's identity matches the
1716    /// checkpoint record. On mismatch, returns a
1717    /// `NonDeterministicExecution` error.
1718    ///
1719    /// `claimed_type` is the operation type string the SDK sends to the
1720    /// backend (e.g. `Step`, `Wait`, `Context`, `ChainedInvoke`,
1721    /// `Callback`).
1722    ///
1723    /// `claimed_sub_type` is the sub-type (e.g. `Step`, `Wait`, `Map`,
1724    /// `RunInChildContext`) or `None` for operations without a sub-type.
1725    ///
1726    /// `claimed_name` is the user-supplied `.name("...")` or `None`.
1727    ///
1728    /// Production paths validate through [`Self::checkpoint_view_validated`]
1729    /// (one read-guard pass, no record clone); this record-taking form is
1730    /// retained as the direct unit-test harness for the identity-matching
1731    /// rules both share.
1732    #[cfg(test)]
1733    pub(crate) fn validate_replay_identity(
1734        &self,
1735        record: &CheckpointRecord,
1736        wire_id: &str,
1737        claimed_type: &str,
1738        claimed_sub_type: Option<&str>,
1739        claimed_name: Option<&str>,
1740    ) -> Result<(), OperationError> {
1741        match replay_identity_mismatch(record, claimed_type, claimed_sub_type, claimed_name) {
1742            None => Ok(()),
1743            Some(expected) => Err(self.replay_mismatch_error(
1744                wire_id,
1745                &expected,
1746                claimed_type,
1747                claimed_sub_type,
1748                claimed_name,
1749            )),
1750        }
1751    }
1752
1753    /// Builds the `NonDeterministicExecution` mismatch error and records it
1754    /// on the shared fatal slot.
1755    ///
1756    /// A replay identity mismatch is execution-fatal: recording it on the
1757    /// shared slot makes the invocation driver fail the execution with the
1758    /// dedicated error even if the returned `Err` is swallowed on its way
1759    /// up: stored as a rejected outcome by `join_all`, out-raced by a
1760    /// sibling's success in `select_ok`, stringified through a
1761    /// child-context boundary, or tolerated by a map/parallel completion
1762    /// config.
1763    fn replay_mismatch_error(
1764        &self,
1765        wire_id: &str,
1766        expected: &str,
1767        claimed_type: &str,
1768        claimed_sub_type: Option<&str>,
1769        claimed_name: Option<&str>,
1770    ) -> OperationError {
1771        let err = OperationError::from_kind(OperationErrorKind::NonDeterministicExecution(
1772            NonDeterministicExecutionError::from_kind(
1773                NonDeterministicExecutionErrorKind::OperationMismatch(
1774                    crate::error::OperationMismatch::new(
1775                        wire_id,
1776                        expected,
1777                        format_op_identity(claimed_type, claimed_sub_type, claimed_name),
1778                    ),
1779                ),
1780            ),
1781        ));
1782        self.inner.coordinator.record_fatal(
1783            "NonDeterministicExecutionError".to_owned(),
1784            crate::error::chain_string(&err),
1785        );
1786        err
1787    }
1788
1789    /// Builds the execution-fatal error for a checkpoint record whose
1790    /// status this SDK version does not recognize, recording it on the
1791    /// shared fatal slot (issue #45).
1792    ///
1793    /// Exactly like [`Self::replay_mismatch_error`], the fatal-slot write
1794    /// makes the invocation driver fail the execution with the dedicated
1795    /// error even if the returned `Err` is swallowed on its way up. The
1796    /// error message carries the raw status verbatim, so the operator can
1797    /// see which service-side status the SDK build predates.
1798    pub(crate) fn unrecognized_status_error(&self, wire_id: &str, raw: &str) -> OperationError {
1799        let err = OperationError::from_kind(OperationErrorKind::NonDeterministicExecution(
1800            NonDeterministicExecutionError::from_kind(
1801                NonDeterministicExecutionErrorKind::UnrecognizedStatus(
1802                    crate::error::UnrecognizedStatus::new(wire_id, raw),
1803                ),
1804            ),
1805        ));
1806        self.inner.coordinator.record_fatal(
1807            "NonDeterministicExecutionError".to_owned(),
1808            crate::error::chain_string(&err),
1809        );
1810        err
1811    }
1812
1813    /// What an operation does when its checkpoint write returned an
1814    /// error. Never returns control to the caller, with one exception.
1815    ///
1816    /// A failed write is recorded where its response arrives (see
1817    /// [`InvocationCoordinator::fail_write`]), and its caller never
1818    /// resolves. So the only error a write returns to its caller is the
1819    /// one of a write that sent nothing because the invocation coordinator
1820    /// was created without a Tokio runtime (see
1821    /// [`Self::spawn_batch_flush`]). The operation then returns the
1822    /// error `.spawn()` returns without a runtime, at once, and nothing is
1823    /// recorded. Any other error leaves the operation pending: the
1824    /// recorded failure ends the drive, and the operation writes no
1825    /// further update.
1826    pub(crate) async fn checkpoint_failure_unrecoverable<T: NoRuntimeOutcome>(
1827        &self,
1828        err: ExecutionClientError,
1829    ) -> T {
1830        if let Some(no_runtime) = std::error::Error::source(&err)
1831            .and_then(|source| source.downcast_ref::<crate::task::NoRuntime>())
1832        {
1833            return T::no_runtime(*no_runtime);
1834        }
1835        std::future::pending::<T>().await
1836    }
1837
1838    /// Records the failed checkpoint write of `op_wire_id` (see
1839    /// [`InvocationCoordinator::fail_write`]): the write-failure latch,
1840    /// and the classification in the fatal slot. The queued writes are
1841    /// dropped. A retryable failure fails the invocation. A non-retryable
1842    /// failure fails the execution with the record of
1843    /// [`checkpoint_execution_failure_wire`](crate::error::checkpoint_execution_failure_wire).
1844    ///
1845    /// [`Self::checkpoint_direct_with_events`] calls it, with the
1846    /// checkpoint-token lock held, and no other code does.
1847    fn record_write_failure(&self, op_wire_id: &str, err: &ExecutionClientError) {
1848        if err.is_retryable() {
1849            tracing::error!(
1850                operation_id = %op_wire_id,
1851                error = %err,
1852                "retryable checkpoint failure exhausted retries; failing the invocation \
1853                 so the service re-invokes"
1854            );
1855        } else {
1856            tracing::error!(
1857                operation_id = %op_wire_id,
1858                error = %err,
1859                "non-retryable checkpoint failure; failing the execution"
1860            );
1861        }
1862        self.inner
1863            .coordinator
1864            .fail_write(checkpoint_failure_fatal(err), || {
1865                self.abandon_queued_writes();
1866            });
1867    }
1868
1869    /// Ends the invocation because a serdes call failed with a marked
1870    /// error. Never resolves.
1871    ///
1872    /// A serdes marks a transient failure with
1873    /// [`RetryableSerdesError`](crate::serdes::RetryableSerdesError). The
1874    /// failure records nothing: the invocation fails with a Lambda runtime
1875    /// error, and the service invokes the function again. The next
1876    /// invocation then repeats the failed serdes call. Like
1877    /// [`Self::checkpoint_failure_unrecoverable`], the fault goes into the
1878    /// fatal slot that every scope shares, so user code can neither catch
1879    /// nor ignore it.
1880    pub(crate) async fn serdes_fault<T>(
1881        &self,
1882        call: &'static str,
1883        operation_id: &str,
1884        name: Option<&str>,
1885        err: BoxError,
1886    ) -> T {
1887        let detail = crate::error::wire_message(&*err);
1888        let message = match name {
1889            Some(name) => {
1890                format!("serdes {call} failed for operation \"{name}\" ({operation_id}): {detail}")
1891            }
1892            None => format!("serdes {call} failed for operation {operation_id}: {detail}"),
1893        };
1894        tracing::error!(
1895            operation_id = %operation_id,
1896            error = %detail,
1897            "serdes {call} failed with a retryable error; failing the invocation so the \
1898             service re-invokes"
1899        );
1900        self.inner
1901            .coordinator
1902            .record_invocation_fault(crate::error::SERDES_FAILED_ERROR_TYPE, message);
1903        std::future::pending::<T>().await
1904    }
1905
1906    /// Ends the invocation when `err` is marked with
1907    /// `RetryableSerdesError`. Returns `err` unchanged otherwise.
1908    ///
1909    /// Every call to a user [`Serdes`](crate::serdes::Serdes) passes its
1910    /// error through this method before any other handling. So an
1911    /// unmarked error takes the caller's permanent-failure path, and a
1912    /// marked error never reaches it.
1913    pub(crate) async fn route_serdes_error(
1914        &self,
1915        call: &'static str,
1916        operation_id: &str,
1917        name: Option<&str>,
1918        err: BoxError,
1919    ) -> BoxError {
1920        if crate::serdes::is_retryable_serdes_error(&*err) {
1921            return self.serdes_fault(call, operation_id, name, err).await;
1922        }
1923        err
1924    }
1925
1926    /// Checkpoints operation updates via the execution client.
1927    ///
1928    /// The updates join the shared coalescing batch, and this call resolves
1929    /// with the batch's result once the batch is written. Without a
1930    /// `checkpoint_delay` the flush is requested at once; with one, after
1931    /// at most the configured delay, or earlier if a flush point drains the
1932    /// buffer.
1933    pub(crate) async fn checkpoint_updates(
1934        &self,
1935        updates: Vec<OperationUpdate>,
1936    ) -> Result<CheckpointOutput, ExecutionClientError> {
1937        self.submit_checkpoint_updates(updates).await
1938    }
1939
1940    /// Starts the checkpoint write of [`Self::checkpoint_updates`] before
1941    /// it returns, and returns the future of its result.
1942    ///
1943    /// The updates join the batch and take their checkpoint permit under
1944    /// the buffer lock. So the write keeps its place in the write order,
1945    /// and continues when the returned value is dropped.
1946    pub(crate) fn submit_checkpoint_updates(
1947        &self,
1948        updates: Vec<OperationUpdate>,
1949    ) -> SubmittedWrite {
1950        self.checkpoint_with_urgency(updates, false)
1951    }
1952
1953    /// Checkpoints operation updates, flushing the coalescing buffer
1954    /// immediately instead of waiting out the delay window.
1955    ///
1956    /// This is the "callback creation" flush point of the
1957    /// [`checkpoint_delay`](crate::OptionsBuilder::checkpoint_delay)
1958    /// contract: the caller needs the backend's response right away (the
1959    /// service assigns the callback ID in it), so the batch, including any
1960    /// previously buffered updates, which keeps write order intact, is
1961    /// written now. Identical to [`Self::submit_checkpoint_updates`] when
1962    /// no delay is configured.
1963    ///
1964    /// The updates join the batch before this returns, as with
1965    /// [`Self::submit_checkpoint_updates`].
1966    pub(crate) fn submit_checkpoint_updates_urgent(
1967        &self,
1968        updates: Vec<OperationUpdate>,
1969    ) -> SubmittedWrite {
1970        self.checkpoint_with_urgency(updates, true)
1971    }
1972
1973    /// Fixes the position of `updates` in the write order, as
1974    /// [`Self::submit_checkpoint_updates`] does, but holds them back until
1975    /// the returned [`ReservedWrite`] is released or dropped.
1976    ///
1977    /// The updates join the open batch, and the batch is not flushed while
1978    /// the reservation holds it. So no request carries them, and no response
1979    /// is merged for them, before the release.
1980    pub(crate) fn reserve_checkpoint_updates(
1981        &self,
1982        updates: Vec<OperationUpdate>,
1983    ) -> ReservedWrite {
1984        #[cfg(test)]
1985        let recorded = updates.clone();
1986        let tracked = self.track_updates(updates);
1987        let coalescer = Arc::clone(&self.inner.coalescer);
1988        let Some(batch) = self.join_batch(&coalescer, tracked, false, None, true) else {
1989            return ReservedWrite::rejected();
1990        };
1991        let write = batch_result(batch).reported_by(self);
1992        #[cfg(test)]
1993        self.record_submission(&recorded, &write);
1994        ReservedWrite {
1995            write,
1996            hold: Some(coalescer),
1997        }
1998    }
1999
2000    /// Sends one checkpoint with no updates for the refresh scheduler.
2001    ///
2002    /// `first_wire_id` travels with the request: a failure of a request
2003    /// with no update is recorded for it (see [`Self::record_write_failure`]).
2004    /// The failure is recorded where the response arrives, on the detached
2005    /// flusher, so it decides the invocation's outcome even when
2006    /// the driver dropped the scheduler, and so this future. A failed
2007    /// refresh never resolves.
2008    pub(crate) async fn checkpoint_refresh(&self, first_wire_id: &str) {
2009        let write = self.write_with_urgency(Vec::new(), true, Some(first_wire_id));
2010        // The task that sends the refresh delivers the change sets queued
2011        // so far once the call returns. The scheduler runs inside a Tokio
2012        // runtime, so the write never returns the error of a write that
2013        // found none, and a failed refresh never resolves.
2014        drop(write.reported_by(self).await);
2015    }
2016
2017    /// Shared body of the buffered checkpoint paths: pairs each update
2018    /// with its captured lifecycle-event metadata, then hands the pairs to
2019    /// the write path, which emits each event as soon as the write that
2020    /// persists its update succeeds. The write starts before this returns.
2021    fn checkpoint_with_urgency(
2022        &self,
2023        updates: Vec<OperationUpdate>,
2024        urgent: bool,
2025    ) -> SubmittedWrite {
2026        // Lifecycle events (see `crate::observability`): every operation
2027        // transition the SDK records passes through here, so this one site
2028        // covers `operation_started` / `operation_succeeded` /
2029        // `operation_failed` / `operation_retry_scheduled` for every
2030        // operation type. The metadata is captured BEFORE the write (the
2031        // checkpoint response mutates the log the attempt is derived from,
2032        // and the updates themselves are consumed by it; the capture also
2033        // snapshots this call's span so the event keeps its originating
2034        // context). Ownership then travels WITH the update into the write
2035        // path, the buffered flush task, not this possibly-cancelled
2036        // future, emits the event, so a contributor dropped after joining
2037        // a batch (a lost `race`/`select_ok` branch) cannot suppress
2038        // telemetry for a transition the flush still persists, and events
2039        // are emitted per persisted chunk rather than per batch. A rejected
2040        // write records nothing, so it emits nothing.
2041        let tracked = self.track_updates(updates);
2042        self.write_with_urgency(tracked, urgent, None)
2043            .reported_by(self)
2044    }
2045
2046    /// Pairs each update with its lifecycle-event metadata, captured now
2047    /// (see [`Self::checkpoint_with_urgency`]).
2048    fn track_updates(&self, updates: Vec<OperationUpdate>) -> Vec<TrackedUpdate> {
2049        updates
2050            .into_iter()
2051            .map(|update| {
2052                // The current attempt is one past the recorded retry count,
2053                // exactly as the step live path derives it.
2054                let attempt = self
2055                    .inner
2056                    .engine
2057                    .checkpoint_log
2058                    .status_view(update.id())
2059                    .map_or(0, |view| view.attempt)
2060                    .saturating_add(1);
2061                TrackedUpdate::capture(update, attempt)
2062            })
2063            .collect()
2064    }
2065
2066    /// Dispatches a checkpoint write: joins the coalescing batch. Under a
2067    /// non-zero delay, the non-urgent join that opens a batch starts a
2068    /// detached flush that waits out the window; an urgent contributor, or
2069    /// any contributor under a zero-delay coalescer (the default), requests
2070    /// the flush right away. Either way the flush task claims and writes the
2071    /// batch under the coalescer's writer lock, so batching still emerges
2072    /// while an earlier write is in flight.
2073    ///
2074    /// Joining the batch transfers ownership of each update's lifecycle
2075    /// event to the coalescer: the flush task that persists the update
2076    /// emits it, so this future can be dropped mid-await without losing
2077    /// telemetry for a transition the flush still records.
2078    ///
2079    /// The joined batch passes `refresh_id` with its write (see
2080    /// [`Self::checkpoint_refresh`]).
2081    ///
2082    /// The join happens before this returns, so the returned
2083    /// [`SubmittedWrite`] only waits for the result.
2084    fn write_with_urgency(
2085        &self,
2086        updates: Vec<TrackedUpdate>,
2087        urgent: bool,
2088        refresh_id: Option<&str>,
2089    ) -> SubmittedWrite {
2090        // The recorder sees the submission before it joins the coalescer,
2091        // so before the flusher can send it. The write
2092        // built below takes the flag the recorder saw.
2093        #[cfg(test)]
2094        let flag = {
2095            let flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
2096            let recorder = self.hooks().submissions.clone();
2097            if let Some(recorder) = recorder
2098                && !updates.is_empty()
2099                && self.admission().admits(self.owners())
2100            {
2101                let submitted: Vec<OperationUpdate> =
2102                    updates.iter().map(|t| t.update.clone()).collect();
2103                recorder(&submitted, &flag);
2104            }
2105            flag
2106        };
2107        #[cfg_attr(not(test), expect(unused_mut))] // reason: only the test build sets the flag
2108        let mut write = self.write_with_urgency_unrecorded(updates, urgent, refresh_id);
2109        #[cfg(test)]
2110        {
2111            write.first_poll = flag;
2112        }
2113        write
2114    }
2115
2116    /// [`Self::write_with_urgency`] without the test submission recorder.
2117    fn write_with_urgency_unrecorded(
2118        &self,
2119        updates: Vec<TrackedUpdate>,
2120        urgent: bool,
2121        refresh_id: Option<&str>,
2122    ) -> SubmittedWrite {
2123        let coalescer = Arc::clone(&self.inner.coalescer);
2124        match self.join_batch(&coalescer, updates, urgent, refresh_id, false) {
2125            Some(batch) => batch_result(batch),
2126            None => SubmittedWrite::rejected(),
2127        }
2128    }
2129
2130    /// Joins `updates` to the coalescer's open batch and schedules its
2131    /// flush. With `reserve`, the join holds the batch until the hold is
2132    /// released (see [`ReservedWrite`]). Returns `None` when admission
2133    /// rejects the join.
2134    fn join_batch(
2135        &self,
2136        coalescer: &Arc<CheckpointCoalescer>,
2137        updates: Vec<TrackedUpdate>,
2138        urgent: bool,
2139        refresh_id: Option<&str>,
2140        reserve: bool,
2141    ) -> Option<Arc<CheckpointBatch>> {
2142        // Admission (see `Admission`): an update from an abandoned owner is
2143        // rejected, and a rejected write never resolves. Its task is being
2144        // cancelled. Returning `Ok` instead would let an operation continue
2145        // as if its update were recorded. The admitted join holds a
2146        // checkpoint permit until the batch's response is merged.
2147        // After a fatal record, a failed write included, nothing more is
2148        // admitted: the join's checkpoint permit is refused under the
2149        // buffer lock (see `InvocationCoordinator::acquire`), so the write
2150        // is rejected before it joins the buffer. A join admitted before
2151        // the record stays in the buffer and is written.
2152        let (batch, opened) = coalescer.join_owned(
2153            self.admission(),
2154            self.coordinator(),
2155            self.owners(),
2156            updates,
2157            refresh_id,
2158            reserve,
2159        )?;
2160        if urgent || coalescer.delay().is_zero() {
2161            self.spawn_batch_flush(coalescer, &batch, Duration::ZERO);
2162        } else if opened {
2163            // The join that opened the batch arms its window on a detached
2164            // task, so the batch is written even when every contributor is
2165            // dropped before the window elapses.
2166            self.spawn_batch_flush(coalescer, &batch, coalescer.delay());
2167        }
2168        Some(batch)
2169    }
2170
2171    /// Writes one request's worth of tracked updates immediately and, if,
2172    /// and only if, the checkpoint call that persists them succeeds, emits
2173    /// each update's lifecycle event (inside the span captured with it).
2174    /// This is the single site that pairs "the transition was persisted"
2175    /// with "its telemetry was emitted": every chunk of a batched write
2176    /// funnels through it. The emission happens
2177    /// inside [`Self::checkpoint_direct_with_events`], immediately after
2178    /// the service accepts the write, before the fallible pagination
2179    /// hydration that follows it, so a transition the service recorded
2180    /// always emits its event even when hydrating the paginated state
2181    /// fails afterwards.
2182    ///
2183    /// The batch flusher and the write after the drive call it, and it runs
2184    /// no admission check: a flushed batch's updates were admitted when they
2185    /// joined the buffer, and the write after the drive comes from the root
2186    /// context. An update without an event emits none.
2187    async fn write_tracked_direct(
2188        &self,
2189        tracked: Vec<TrackedUpdate>,
2190    ) -> Result<CheckpointOutput, WriteError> {
2191        let mut updates = Vec::with_capacity(tracked.len());
2192        let mut events = Vec::with_capacity(tracked.len());
2193        for TrackedUpdate { update, event } in tracked {
2194            updates.push(update);
2195            events.extend(event);
2196        }
2197        self.write_unchecked(updates, &events, None).await
2198    }
2199
2200    /// Writes updates that skip the admission check: those of a flushed
2201    /// batch and of the write after the drive. `refresh_id` is the batch's
2202    /// (see [`Self::checkpoint_refresh`]).
2203    async fn write_unchecked(
2204        &self,
2205        updates: Vec<OperationUpdate>,
2206        events: &[crate::tracing_layer::PendingTransitionEvent],
2207        refresh_id: Option<&str>,
2208    ) -> Result<CheckpointOutput, WriteError> {
2209        self.checkpoint_direct_with_events(updates, events, refresh_id)
2210            .await
2211    }
2212
2213    /// Spawns a task that claims `batch` (if it is still the open batch)
2214    /// and writes its buffered updates, publishing the result to every
2215    /// contributor. The claim and the write both happen while holding the
2216    /// coalescer's writer lock, so buffered writes are totally ordered and
2217    /// a flush point that acquires the lock waits for this write to finish.
2218    /// A write that sent nothing or failed publishes nothing, so the
2219    /// batch's contributors never resolve: the checkpoint token was
2220    /// withdrawn, or this write or an earlier one failed, and the failure
2221    /// is recorded where its response arrived (see
2222    /// [`Self::record_write_failure`]).
2223    ///
2224    /// The write runs on its own task deliberately: a contributor dropped
2225    /// mid-await (a lost `race`, a dropped `DurableFuture`) must not cancel
2226    /// an in-flight batch write that other contributors are waiting on.
2227    fn spawn_batch_flush(
2228        &self,
2229        coalescer: &Arc<CheckpointCoalescer>,
2230        batch: &Arc<CheckpointBatch>,
2231        delay: Duration,
2232    ) {
2233        let ctx = self.clone();
2234        let claim_from = Arc::clone(coalescer);
2235        let coalescer = Arc::clone(coalescer);
2236        let publish_to = Arc::clone(batch);
2237        let batch = Arc::clone(batch);
2238        // Detached mode of the task gateway (src/task.rs): this internal
2239        // task runs no handler code and creates no durable operations, and it
2240        // must survive a cancelled contributor. It is still counted, so the
2241        // invocation waits for an in-flight write before it returns. The
2242        // batch's checkpoint permits keep the invocation from suspending
2243        // until the response is merged and published.
2244        // A panic of the checkpoint client unwinds the flush, and nothing
2245        // publishes the batch. The task gateway records the panic on the
2246        // coordinator, which ends the drive. The contributors stay pending
2247        // until the drive's cleanup cancels them.
2248        let flush = async move {
2249            if !delay.is_zero() {
2250                // The window ends early when the invocation would suspend
2251                // but for buffered writes, or when the drive ended. A
2252                // counted task that slept out the whole window would keep
2253                // a resolved handler from returning for up to the full
2254                // `checkpoint_delay`, which has no upper bound.
2255                ctx.coordinator().coalescing_window(delay).await;
2256            }
2257            let _writer = coalescer.writer_lock().lock().await;
2258            if let Some((updates, permits)) = coalescer.take_batch_unheld(&batch).await {
2259                // The merge already woke the operations the response
2260                // readied. The contributors' wakes take their own permits
2261                // before the batch's permits are released.
2262                if let Ok(output) = ctx
2263                    .write_batched_updates(updates, coalescer.limits(), batch.refresh_id())
2264                    .await
2265                {
2266                    batch.publish(Ok(output));
2267                }
2268                drop(permits);
2269            }
2270        };
2271        let spawned = crate::task::spawn_detached(self.coordinator(), flush);
2272        // The invocation coordinator was created without a Tokio runtime, so
2273        // nothing can write the batch: it is dropped from the buffer with its updates and
2274        // permits, so no later flush sends it, and every contributor
2275        // receives the error instead of waiting forever. Its source marks
2276        // it, so an operation returns the error `.spawn()` returns without
2277        // a runtime (see `checkpoint_failure_unrecoverable`). When another
2278        // flusher already claimed the batch, that flusher publishes its
2279        // result instead. Nothing was sent, and nothing is recorded. After
2280        // a failed write the batch is not claimed and nothing is published
2281        // (see `CheckpointCoalescer::claim_unspawned`).
2282        let Err(no_runtime) = spawned else {
2283            return;
2284        };
2285        #[cfg(test)]
2286        crate::coordinator::race_points::reach(
2287            crate::coordinator::race_points::RacePoint::BatchNoRuntimeClaim,
2288        );
2289        if let Some((updates, permits)) =
2290            claim_from.claim_unspawned(&publish_to, self.coordinator())
2291        {
2292            drop(updates);
2293            publish_to.publish(Err(ExecutionClientError::with_source(
2294                ExecutionClientErrorScope::Invocation,
2295                no_runtime.to_string(),
2296                no_runtime,
2297            )));
2298            drop(permits);
2299        }
2300    }
2301
2302    /// Writes a sealed batch, splitting it into request-sized chunks that
2303    /// respect the coalescer's [`BatchLimits`] (operation count and
2304    /// estimated payload bytes) while preserving join order. Returns the
2305    /// last chunk's output on success: backend-assigned fields from every
2306    /// chunk are already merged into the checkpoint log. The first chunk
2307    /// that sends nothing or fails stops the batch: the chunks after it are
2308    /// not sent. `refresh_id` is the batch's (see
2309    /// [`Self::checkpoint_refresh`]).
2310    ///
2311    /// Each chunk's lifecycle events are emitted immediately after that
2312    /// chunk's write succeeds (see [`Self::write_tracked_direct`]): a chunk
2313    /// persisted before a later chunk fails still emits its events, so
2314    /// telemetry stays faithful to what was actually recorded.
2315    ///
2316    /// Callers must hold the coalescer's writer lock (see the invariants in
2317    /// [`crate::checkpoint_coalescer`]).
2318    async fn write_batched_updates(
2319        &self,
2320        updates: Vec<TrackedUpdate>,
2321        limits: BatchLimits,
2322        refresh_id: Option<String>,
2323    ) -> Result<CheckpointOutput, WriteError> {
2324        if updates.is_empty() {
2325            // A refresh joins with no update, and its request carries none.
2326            return self
2327                .write_unchecked(Vec::new(), &[], refresh_id.as_deref())
2328                .await;
2329        }
2330        let mut last = None;
2331        for chunk in split_into_requests(updates, &limits) {
2332            last = Some(self.write_tracked_direct(chunk).await?);
2333        }
2334        #[expect(clippy::expect_used)]
2335        // reason: a non-empty batch splits into at least one request.
2336        Ok(last.expect("a non-empty batch has a request"))
2337    }
2338
2339    /// Unconditionally drains the checkpoint coalescing buffer, writing any
2340    /// pending batches now, and waits for any in-flight batch write to
2341    /// finish before returning.
2342    ///
2343    /// This is the "suspension" and "execution completion" flush point of
2344    /// the [`checkpoint_delay`](crate::OptionsBuilder::checkpoint_delay)
2345    /// contract: the invocation wrapper calls it after the driver settles,
2346    /// before reporting `PENDING`, `SUCCEEDED`, or `FAILED` to the service,
2347    /// so a checkpoint that must land before the invocation ends is never
2348    /// held back by the buffer. Because every claimed batch is written while
2349    /// holding the coalescer's writer lock, acquiring that lock here makes
2350    /// this a true barrier: a batch a delay timer already claimed cannot
2351    /// still be in flight when this returns. A no-op when the buffer is
2352    /// idle.
2353    ///
2354    /// A write that sends nothing or fails stops the drain: the checkpoint
2355    /// token was withdrawn, or this write or an earlier one failed, and
2356    /// the failure is recorded where its response arrived (see
2357    /// [`Self::record_write_failure`]). The batch's contributors never
2358    /// resolve, and the rest of the buffer was dropped with the queued
2359    /// writes.
2360    pub(crate) async fn flush_pending_checkpoints(&self) {
2361        let coalescer = &self.inner.coalescer;
2362        // Acquiring the writer lock waits out any in-flight batch write
2363        // (batches are only claimed and written under it), then holding it
2364        // across the drain keeps this flush ordered after those writes.
2365        let _writer = coalescer.writer_lock().lock().await;
2366        while let Some((batch, updates, permits)) = coalescer.take_any_unheld().await {
2367            let written = self
2368                .write_batched_updates(updates, coalescer.limits(), batch.refresh_id())
2369                .await;
2370            drop(permits);
2371            match written {
2372                Ok(output) => batch.publish(Ok(output)),
2373                Err(_) => break,
2374            }
2375        }
2376    }
2377
2378    /// Writes operation updates through the coalescer as an urgent write,
2379    /// and delivers the change sets on the caller's task once the flush task
2380    /// returned the result.
2381    #[cfg(test)]
2382    pub(crate) async fn checkpoint_updates_direct(
2383        &self,
2384        updates: Vec<OperationUpdate>,
2385    ) -> Result<CheckpointOutput, ExecutionClientError> {
2386        let result = self.submit_checkpoint_updates_urgent(updates).await;
2387        self.report_operation_changes().await;
2388        result
2389    }
2390
2391    /// A write after the drive ended, when no checkpoint permit can be
2392    /// taken: the execution's own update for an oversized result.
2393    ///
2394    /// The drive's cleanup waited for every SDK task, so the write runs on
2395    /// the caller's task. It takes the coalescer's writer lock and writes
2396    /// its updates as a drained batch is written, in requests cut to the
2397    /// coalescer's limits, and its updates emit no lifecycle event. It
2398    /// returns its result to the caller at once: [`WriteError::Stopped`]
2399    /// when the token is or becomes withdrawn, or when a write failed
2400    /// before, and [`WriteError::Failed`] when it fails itself.
2401    pub(crate) async fn checkpoint_updates_after_drive(
2402        &self,
2403        updates: Vec<OperationUpdate>,
2404    ) -> Result<CheckpointOutput, WriteError> {
2405        let coalescer = &self.inner.coalescer;
2406        let _writer = coalescer.writer_lock().lock().await;
2407        let tracked = updates.into_iter().map(TrackedUpdate::untracked).collect();
2408        self.write_batched_updates(tracked, coalescer.limits(), None)
2409            .await
2410    }
2411
2412    /// Writes one request-sized chunk of a batch and owns the lifecycle
2413    /// events paired with its updates. Each event is emitted
2414    /// the moment `client.checkpoint` returns success, the point at which
2415    /// the service has durably recorded the transitions, and **before**
2416    /// the pagination hydration below, which can fail after the write
2417    /// already persisted. Emitting first keeps telemetry faithful to what
2418    /// was recorded: a rejected write emits nothing, and a persisted write
2419    /// emits everything even when the follow-up `get_state` fetch fails.
2420    ///
2421    /// It runs no admission check. The batch flusher's updates were
2422    /// admitted when they joined the buffer, and a write after the drive
2423    /// comes from the root context, which no scope encloses.
2424    ///
2425    /// After the checkpoint token was withdrawn (see
2426    /// [`InvocationCoordinator::withdraw`]), or after a failed write, the
2427    /// call sends nothing and returns [`WriteError::Stopped`].
2428    ///
2429    /// A failed call is recorded here (see [`Self::record_write_failure`]),
2430    /// for the wire ID of the request's first update, or `refresh_id` for a
2431    /// refresh, which sends none: the transport's error and the error of
2432    /// the state read that follows an accepted paginated write, under the
2433    /// checkpoint-token lock (see [`Self::write_locked`]), and a missing
2434    /// execution client.
2435    async fn checkpoint_direct_with_events(
2436        &self,
2437        updates: Vec<OperationUpdate>,
2438        events: &[crate::tracing_layer::PendingTransitionEvent],
2439        refresh_id: Option<&str>,
2440    ) -> Result<CheckpointOutput, WriteError> {
2441        #[cfg(test)]
2442        self.pass_gate(&updates).await;
2443        let failure_id = updates
2444            .first()
2445            .map(|update| update.id().to_owned())
2446            .or_else(|| refresh_id.map(str::to_owned))
2447            .unwrap_or_default();
2448        // Hold the async mutex across the entire checkpoint call to
2449        // serialize concurrent branch checkpoints. This is the Rust
2450        // equivalent of Go's sync.Mutex held across the blocking API call
2451        // in checkpointer.checkpoint().
2452        let token_guard = self.inner.checkpoint_token.lock().await;
2453        // Both latches are set under this lock, so a request that waited
2454        // for the lock behind the withdrawn or failed one sees them here.
2455        if self.inner.coordinator.withdrawn() || self.inner.coordinator.write_failed() {
2456            return Err(WriteError::Stopped);
2457        }
2458        let Some(client) = self.inner.execution_client.as_ref() else {
2459            // Recorded under the checkpoint-token lock, as a failed call
2460            // is, so it is ordered with every other write.
2461            let err = ExecutionClientError::new_non_retryable("no execution client configured");
2462            self.record_write_failure(&failure_id, &err);
2463            drop(token_guard);
2464            return Err(WriteError::Failed(err));
2465        };
2466        self.write_locked(client, token_guard, updates, events, &failure_id)
2467            .await
2468    }
2469
2470    /// The write of [`Self::checkpoint_direct_with_events`], with the
2471    /// checkpoint-token lock held. A failed call or a failed state read
2472    /// after it is recorded for `failure_id` before the lock is released.
2473    async fn write_locked(
2474        &self,
2475        client: &Arc<dyn ExecutionClient>,
2476        mut token_guard: tokio::sync::MutexGuard<'_, String>,
2477        updates: Vec<OperationUpdate>,
2478        events: &[crate::tracing_layer::PendingTransitionEvent],
2479        failure_id: &str,
2480    ) -> Result<CheckpointOutput, WriteError> {
2481        let retried = wfc_retries(&updates);
2482        // A tokenless response to a request with an `EXECUTION` update is
2483        // not a withdrawal, as in the JS and Python SDKs.
2484        let has_execution_update = updates
2485            .iter()
2486            .any(|update| update.r#type() == &aws_sdk_lambda::types::OperationType::Execution);
2487        let output = match client
2488            .checkpoint(&self.inner.execution_arn, &token_guard, updates)
2489            .await
2490        {
2491            Ok(output) => output,
2492            Err(err) => {
2493                self.record_write_failure(failure_id, &err);
2494                return Err(WriteError::Failed(err));
2495            }
2496        };
2497        let next_token = output.checkpoint_token().map(ToOwned::to_owned);
2498        let tokenless = next_token.is_none();
2499
2500        // The service has recorded the transitions: emit their lifecycle
2501        // events now, before the fallible pagination hydration below can
2502        // turn an already-persisted write into a caller-visible error.
2503        for event in events {
2504            event.emit(self.execution_arn(), &self.lambda_context().request_id);
2505        }
2506
2507        match next_token {
2508            // Rotate the token while still holding the lock.
2509            Some(next_token) => *token_guard = next_token,
2510            None if has_execution_update => {}
2511            None => {
2512                // The service accepts no further checkpoints from this
2513                // invocation. The response's state is not merged, and the
2514                // request's callers are not resolved. The latch is set
2515                // under the token lock, the queued writes are dropped
2516                // unresolved, and a request that already waits for the
2517                // lock sees the latch before it is sent.
2518                self.inner.coordinator.withdraw(
2519                    self.execution_arn(),
2520                    &self.lambda_context().request_id,
2521                    || self.abandon_queued_writes(),
2522                );
2523                return Err(WriteError::Stopped);
2524            }
2525        }
2526
2527        // If the checkpoint response is paginated, read the remaining pages
2528        // via get_state before anything is merged, so that the merges and
2529        // the queue below run under one hold of the publication guard,
2530        // which no `.await` crosses. The token lock stays held through this
2531        // read: releasing it first would let a concurrent branch checkpoint
2532        // and rotate the token, leaving this get_state call with a stale
2533        // token. A tokenless response to an `EXECUTION` update left the
2534        // spent token in place, so its pages are not read, as in the JS and
2535        // Python SDKs.
2536        let pages = if output.next_marker.is_some() && !tokenless {
2537            Some(
2538                crate::client::get_full_state(
2539                    client.as_ref(),
2540                    &self.inner.execution_arn,
2541                    &token_guard,
2542                )
2543                .await,
2544            )
2545        } else {
2546            None
2547        };
2548
2549        // Merge the response's operations, then its pages, into the
2550        // checkpoint log so that subsequent reads (e.g. reading callback_id
2551        // after START) see backend-assigned fields, and queue their change
2552        // set, all under the publication guard. A waiting operation reads
2553        // its record's status under the guard's read side (see
2554        // `wait_for_record`), so it sees a status this response changed
2555        // only after the change set is queued, and the change comes before
2556        // its end hook.
2557        let publication = self.inner.engine.checkpoint_log.publish();
2558        let mut merged: Vec<String> = Vec::new();
2559        // The change set of this call, for `on_operation_change`: the wire
2560        // IDs whose status changed, each once, and the `EXECUTION`
2561        // operations that changed the plugins' slot. Only collected with
2562        // plugins.
2563        let mut change_set = ChangeSet::default();
2564        if !output.updated_operations.is_empty() {
2565            let changed_ids = crate::client::merge_operations_into_log(
2566                &self.inner.engine.checkpoint_log,
2567                &output.updated_operations,
2568            );
2569            self.collect_changes(&mut change_set, changed_ids, &output.updated_operations);
2570            merged.extend(
2571                output
2572                    .updated_operations
2573                    .iter()
2574                    .map(|op| op.id().to_owned()),
2575            );
2576        }
2577        match pages {
2578            Some(Ok(full_state)) => {
2579                if !full_state.is_empty() {
2580                    let changed_ids = crate::client::merge_operations_into_log(
2581                        &self.inner.engine.checkpoint_log,
2582                        &full_state,
2583                    );
2584                    self.collect_changes(&mut change_set, changed_ids, &full_state);
2585                    merged.extend(full_state.iter().map(|op| op.id().to_owned()));
2586                }
2587            }
2588            Some(Err(err)) => {
2589                // The response is merged, so its changes are reported even
2590                // though the pages failed.
2591                self.queue_changes(change_set);
2592                drop(publication);
2593                self.record_write_failure(failure_id, &err);
2594                return Err(WriteError::Failed(err));
2595            }
2596            None => {}
2597        }
2598        // Still under the token lock, so change sets queue in response
2599        // order. No hook runs here: a drain point delivers them later.
2600        self.queue_changes(change_set);
2601        drop(publication);
2602
2603        // A `wait_for_condition` RETRY the response did not carry: the log
2604        // takes the status the service recorded. This runs under the token
2605        // lock, so a later response, which carries a newer status, merges
2606        // after it.
2607        for (wire_id, payload) in &retried {
2608            if !merged.iter().any(|id| id == wire_id) {
2609                mark_retry_pending(
2610                    &self.inner.engine.checkpoint_log,
2611                    wire_id,
2612                    payload.as_deref(),
2613                );
2614            }
2615        }
2616
2617        // The token is rotated and the response merged: release the lock,
2618        // then wake every operation the response carries. The write's
2619        // checkpoint permit is still held, so each woken operation holds
2620        // its own permit before this write's permit is released.
2621        drop(token_guard);
2622        self.inner.coordinator.wake_operations(&merged);
2623
2624        Ok(output)
2625    }
2626
2627    /// Drops every queued checkpoint write without resolving its caller:
2628    /// the coalescer's open batch. Its permits are released after the
2629    /// buffer lock is released. The withdrawal of the checkpoint token and
2630    /// a failed write call it (see [`InvocationCoordinator::withdraw`] and
2631    /// [`InvocationCoordinator::fail_write`]).
2632    fn abandon_queued_writes(&self) {
2633        drop(self.inner.coalescer.abandon_pending());
2634    }
2635
2636    /// Adds the changes of one merge to `change_set`: the wire IDs in
2637    /// `changed_ids` not listed yet, and each `EXECUTION` operation of
2638    /// `operations`, which replaces the plugins' slot and is a change when
2639    /// the slot held no operation or one with another status. Does nothing
2640    /// without plugins.
2641    fn collect_changes(
2642        &self,
2643        change_set: &mut ChangeSet,
2644        changed_ids: Vec<String>,
2645        operations: &[aws_sdk_lambda::types::Operation],
2646    ) {
2647        let Some(plugins) = self.plugins() else {
2648            return;
2649        };
2650        for id in changed_ids {
2651            if !change_set.ids.contains(&id) {
2652                change_set.ids.push(id);
2653            }
2654        }
2655        for info in operations
2656            .iter()
2657            .filter_map(crate::client::execution_info_from_operation)
2658        {
2659            if plugins.replace_execution(info.clone()) {
2660                change_set.executions.retain(|held| held.id() != info.id());
2661                change_set.executions.push(info);
2662            }
2663        }
2664    }
2665
2666    /// Queues `change_set` for `on_operation_change` when it lists anything.
2667    /// Each listed operation's info is built from its borrowed record, so
2668    /// no record is cloned and the full log is not projected.
2669    fn queue_changes(&self, change_set: ChangeSet) {
2670        let Some(plugins) = self.plugins() else {
2671            return;
2672        };
2673        let log = &self.inner.engine.checkpoint_log;
2674        let mut updated: Vec<crate::plugin::OperationInfo> = change_set
2675            .ids
2676            .iter()
2677            .filter_map(|id| {
2678                log.with_record(id, |record| {
2679                    crate::plugin::OperationInfo::from_record(id, record)
2680                })
2681            })
2682            .collect();
2683        updated.extend(change_set.executions);
2684        if !updated.is_empty() {
2685            plugins.push_operation_change(updated);
2686        }
2687    }
2688
2689    /// Returns the parent wire ID if this is a child context.
2690    pub(crate) fn parent_wire_id(&self) -> Option<&str> {
2691        self.inner.parent_wire_id.as_deref()
2692    }
2693
2694    /// Returns the attempt number for the given operation from the checkpoint
2695    /// log (0 if no prior attempt recorded).
2696    pub(crate) fn get_attempt(&self, op_id: &OperationId) -> u32 {
2697        // On re-invocation after a RETRY, the backend returns the operation
2698        // with step_details.attempt set to the last completed attempt number.
2699        self.checkpoint_status_view(op_id.positional())
2700            .map_or(0, |view| view.attempt)
2701    }
2702
2703    /// Returns the execution ARN identifying this durable execution.
2704    ///
2705    /// The ARN uniquely identifies the execution across invocations and is
2706    /// stable for the lifetime of the orchestration.
2707    ///
2708    /// # Examples
2709    ///
2710    /// ```no_run
2711    /// use aws_durable_execution_sdk as durable;
2712    ///
2713    /// async fn handler(
2714    ///     _event: serde_json::Value,
2715    ///     ctx: durable::DurableContext,
2716    /// ) -> Result<(), durable::BoxError> {
2717    ///     tracing::info!(arn = ctx.execution_arn(), "starting execution");
2718    ///     Ok(())
2719    /// }
2720    /// ```
2721    pub fn execution_arn(&self) -> &str {
2722        &self.inner.execution_arn
2723    }
2724
2725    /// Returns a reference to the Lambda invocation context.
2726    ///
2727    /// Provides access to the request ID, deadline, function ARN, and other
2728    /// Lambda metadata for the current invocation.
2729    ///
2730    /// # Examples
2731    ///
2732    /// ```no_run
2733    /// use aws_durable_execution_sdk as durable;
2734    ///
2735    /// async fn handler(
2736    ///     _event: serde_json::Value,
2737    ///     ctx: durable::DurableContext,
2738    /// ) -> Result<(), durable::BoxError> {
2739    ///     let request_id = &ctx.lambda_context().request_id;
2740    ///     tracing::info!(%request_id, "invoked");
2741    ///     Ok(())
2742    /// }
2743    /// ```
2744    pub fn lambda_context(&self) -> &lambda_runtime::Context {
2745        &self.inner.lambda_context
2746    }
2747
2748    /// Returns whether the current invocation is in replay mode.
2749    ///
2750    /// During replay, previously-checkpointed operation results are returned
2751    /// without re-execution. Replay mode lasts while the next operation to
2752    /// be claimed was already claimed by a prior invocation, including a
2753    /// started-but-unfinished child context, map, or parallel parent that
2754    /// the resumed invocation re-enters to replay its nested operations.
2755    /// User code can use this flag to suppress duplicate side effects
2756    /// (e.g., logging) during replay.
2757    ///
2758    /// # Examples
2759    ///
2760    /// ```no_run
2761    /// use aws_durable_execution_sdk as durable;
2762    ///
2763    /// async fn handler(
2764    ///     _event: serde_json::Value,
2765    ///     ctx: durable::DurableContext,
2766    /// ) -> Result<(), durable::BoxError> {
2767    ///     if !ctx.is_replaying() {
2768    ///         tracing::info!("first execution, not replay");
2769    ///     }
2770    ///     Ok(())
2771    /// }
2772    /// ```
2773    pub fn is_replaying(&self) -> bool {
2774        self.inner.engine.is_replaying()
2775    }
2776
2777    /// Creates a durable step operation.
2778    ///
2779    /// The step body receives a [`StepContext`] and returns a result. The
2780    /// result is checkpointed on success; on failure the retry strategy
2781    /// determines whether to retry.
2782    ///
2783    /// The step body is `FnOnce + Clone`. The SDK keeps the original closure
2784    /// and calls a fresh clone of it for each attempt. Each capture follows
2785    /// its type's `Clone` implementation: a cloned `Vec` or `String` has its
2786    /// own contents, and a cloned `Arc` shares its allocation.
2787    ///
2788    /// The operation ID is claimed synchronously at this call site.
2789    ///
2790    /// # Examples
2791    ///
2792    /// ```no_run
2793    /// use aws_durable_execution_sdk as durable;
2794    ///
2795    /// async fn handler(
2796    ///     _event: serde_json::Value,
2797    ///     ctx: durable::DurableContext,
2798    /// ) -> Result<String, durable::BoxError> {
2799    ///     let result = ctx.step(|_step_ctx| async {
2800    ///         Ok("computed value".to_owned())
2801    ///     }).name("compute")
2802    ///       .await?;
2803    ///     Ok(result)
2804    /// }
2805    /// ```
2806    pub fn step<O, F, Fut>(&self, f: F) -> StepBuilder<O, F, Fut>
2807    where
2808        F: FnOnce(StepContext) -> Fut + Clone + Send + 'static,
2809        Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
2810        O: Send + 'static,
2811    {
2812        let op_id = self.claim_operation_id("step");
2813        // The closure is stored unerased; the single erasure point is the
2814        // builder's `.future()` / `.await`, producing one DurableFuture box.
2815        StepBuilder::new_internal(self.clone(), op_id, f)
2816    }
2817
2818    /// Creates a durable wait (timer) operation.
2819    ///
2820    /// The operation waits for the specified duration. The invocation
2821    /// suspends during the wait only when no other SDK work can make
2822    /// progress, and a wait that ends while other SDK work runs continues
2823    /// in the same invocation. The wait is checkpointed, so replay does not
2824    /// re-wait.
2825    ///
2826    /// The duration is rounded up to whole seconds, so the wait never ends
2827    /// early. A duration under one second, including [`Duration::ZERO`],
2828    /// waits one second, because the service rejects a zero-second wait.
2829    ///
2830    /// # Examples
2831    ///
2832    /// ```no_run
2833    /// use aws_durable_execution_sdk as durable;
2834    /// use std::time::Duration;
2835    ///
2836    /// async fn handler(
2837    ///     _event: serde_json::Value,
2838    ///     ctx: durable::DurableContext,
2839    /// ) -> Result<(), durable::BoxError> {
2840    ///     ctx.wait(Duration::from_secs(60))
2841    ///         .name("cooldown")
2842    ///         .await?;
2843    ///     Ok(())
2844    /// }
2845    /// ```
2846    pub fn wait(&self, duration: Duration) -> WaitBuilder {
2847        // Round up to whole seconds. The service rejects `WaitSeconds: 0`,
2848        // so the clamp raises a sub-second duration to one second.
2849        #[expect(clippy::cast_possible_truncation)]
2850        // reason: the clamp bounds the value to `1..=i32::MAX` before the cast to i32
2851        let secs = (duration.as_secs_f64().ceil() as i64).clamp(1, i64::from(i32::MAX)) as i32;
2852        let op_id = self.claim_operation_id("wait");
2853        WaitBuilder::new_internal(self.clone(), op_id, secs)
2854    }
2855
2856    /// Creates a durable invoke operation to call another durable function.
2857    ///
2858    /// The output type `O` is the only type parameter: name it with
2859    /// `ctx.invoke::<Receipt>(...)`, or let a typed binding or the
2860    /// function's return type set it. The input needs only
2861    /// `Serialize + Send + 'static`.
2862    ///
2863    /// # Examples
2864    ///
2865    /// ```no_run
2866    /// use aws_durable_execution_sdk as durable;
2867    /// use serde::{Deserialize, Serialize};
2868    ///
2869    /// #[derive(Serialize)]
2870    /// struct ChargeInput { amount: u64 }
2871    ///
2872    /// #[derive(Serialize, Deserialize)]
2873    /// struct Receipt { id: String }
2874    ///
2875    /// async fn handler(
2876    ///     _event: serde_json::Value,
2877    ///     ctx: durable::DurableContext,
2878    /// ) -> Result<String, durable::BoxError> {
2879    ///     let receipt = ctx.invoke::<Receipt>("payment-fn", ChargeInput { amount: 100 })
2880    ///         .name("charge")
2881    ///         .await?;
2882    ///     Ok(receipt.id)
2883    /// }
2884    /// ```
2885    ///
2886    /// The returned builder's input type is opaque and borrows nothing, so a
2887    /// builder stored in a binding works after the context it came from is
2888    /// moved or dropped. The opaque type still names the `&self` lifetime,
2889    /// so a function that takes the context by value cannot return the
2890    /// un-awaited builder:
2891    ///
2892    /// ```compile_fail,E0597
2893    /// use aws_durable_execution_sdk as durable;
2894    /// use durable::builders::InvokeBuilder;
2895    ///
2896    /// fn charge(ctx: durable::DurableContext) -> InvokeBuilder<u32, impl serde::Serialize + Send + 'static> {
2897    ///     ctx.invoke::<u32>("payment-fn", 100_u32)
2898    /// }
2899    /// ```
2900    ///
2901    /// Such a function returns `.future()` instead:
2902    ///
2903    /// ```no_run
2904    /// use aws_durable_execution_sdk as durable;
2905    ///
2906    /// fn charge(ctx: durable::DurableContext) -> durable::DurableFuture<u32> {
2907    ///     ctx.invoke::<u32>("payment-fn", 100_u32).future()
2908    /// }
2909    /// ```
2910    pub fn invoke<O>(
2911        &self,
2912        function_id: impl Into<String>,
2913        input: impl Serialize + Send + 'static,
2914    ) -> InvokeBuilder<O, impl Serialize + Send + 'static>
2915    where
2916        O: Send + 'static,
2917    {
2918        let op_id = self.claim_operation_id("invoke");
2919        // The input is carried TYPED into the builder: the payload serdes
2920        // receives the owned value directly at execution time (a write-only
2921        // transfer), so no intermediate representation is constructed here.
2922        InvokeBuilder::new_internal(self.clone(), op_id, function_id.into(), input)
2923    }
2924
2925    /// Creates a child context for fan-out / sub-orchestration.
2926    ///
2927    /// The child closure receives its own [`DurableContext`] with an
2928    /// independent operation-ID namespace. Use `.spawn()` for eager
2929    /// (background) execution.
2930    ///
2931    /// The body should run at least one durable operation. The child's
2932    /// result is saved, so a replay returns it without running the body,
2933    /// with three exceptions. A body that returns an error is not retried,
2934    /// as a step is. A result over 256 KiB makes the body run again on
2935    /// every replay, to rebuild the result; its record holds the payload of
2936    /// [`ChildBuilder::summary`] instead. A context that
2937    /// [`plugin_child_operations_depth`](crate::OptionsBuilder::plugin_child_operations_depth)
2938    /// preserves also runs its body again on every replay. So put plain
2939    /// computation in a step.
2940    ///
2941    /// To change the error a child returns, call
2942    /// [`map_err`](Result::map_err) on the awaited result.
2943    ///
2944    /// # Examples
2945    ///
2946    /// ```no_run
2947    /// use aws_durable_execution_sdk as durable;
2948    ///
2949    /// async fn handler(
2950    ///     _event: serde_json::Value,
2951    ///     ctx: durable::DurableContext,
2952    /// ) -> Result<String, durable::BoxError> {
2953    ///     let result = ctx.run_in_child_context(|child_ctx| async move {
2954    ///         let v = child_ctx.step(|_| async { Ok(42) }).await?;
2955    ///         Ok(v.to_string())
2956    ///     }).name("branch")
2957    ///       .await?;
2958    ///     Ok(result)
2959    /// }
2960    /// ```
2961    pub fn run_in_child_context<O, F, Fut>(&self, f: F) -> ChildBuilder<O, F, Fut>
2962    where
2963        F: FnOnce(DurableContext) -> Fut + Send + 'static,
2964        Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
2965        O: Send + 'static,
2966    {
2967        let op_id = self.claim_operation_id("run_in_child_context");
2968        // The closure is stored unerased; the child execution converts the
2969        // BoxError into the internal child-error carrier at the boundary,
2970        // and the single erasure point is the builder's `.future()` /
2971        // `.await`, producing one DurableFuture box.
2972        ChildBuilder::new_internal(self.clone(), op_id, f)
2973    }
2974
2975    /// Runs a closure against a child context and retries the closure's
2976    /// **overall** outcome as a unit.
2977    ///
2978    /// Where a step's retry strategy re-runs one operation, `with_retry`
2979    /// re-runs a whole block: if any operation in the block fails (or the
2980    /// closure returns an error), the retry strategy decides whether to run
2981    /// the entire block again. Each attempt receives a **fresh child
2982    /// operation namespace**, so operations recorded by a failed attempt
2983    /// are never replayed into the next one: every operation in the block
2984    /// re-runs on retry. The block waits for the delay between attempts
2985    /// (the backend owns the timer, exactly as step retries do). The
2986    /// invocation suspends during that wait only when no other SDK work can
2987    /// make progress, and a wait that ends while other SDK work runs
2988    /// continues in the same invocation. The retry progress is derived from
2989    /// checkpointed results, so it survives suspension and replays
2990    /// deterministically.
2991    ///
2992    /// The closure is `Fn` rather than `FnOnce` because the SDK calls it
2993    /// once per attempt. It receives the attempt's context and the 1-based
2994    /// attempt number, which is `N` in the attempt's name `attempt-N`. The
2995    /// number is metadata for names and logs: a resumed attempt receives
2996    /// the same number, and operation IDs do not depend on it. Configure
2997    /// the policy with
2998    /// [`retry_strategy`](WithRetryBuilder::retry_strategy) or
2999    /// [`retry_strategy_config`](WithRetryBuilder::retry_strategy_config);
3000    /// without one, the step default applies (6 total attempts with
3001    /// exponential backoff).
3002    ///
3003    /// When retries exhaust, the block fails with a
3004    /// [`ChildContextError`](crate::ChildContextError) of kind
3005    /// `ChildFailed` whose source is the last attempt's
3006    /// [`OperationError`](crate::OperationError).
3007    ///
3008    /// # Examples
3009    ///
3010    /// ```no_run
3011    /// use aws_durable_execution_sdk as durable;
3012    /// use aws_durable_execution_sdk::RetryDecision;
3013    /// use std::time::Duration;
3014    ///
3015    /// async fn handler(
3016    ///     _event: serde_json::Value,
3017    ///     ctx: durable::DurableContext,
3018    /// ) -> Result<String, durable::BoxError> {
3019    ///     // If either step fails, BOTH re-run on the next attempt.
3020    ///     let result = ctx.with_retry(|child, attempt| async move {
3021    ///         tracing::info!(attempt, "reserving and booking");
3022    ///         let quote = child
3023    ///             .step(|_| async { Ok("quote-17".to_owned()) })
3024    ///             .name("reserve-quote")
3025    ///             .await?;
3026    ///         let receipt = child
3027    ///             .step(move |_| async move { Ok(format!("booked:{quote}")) })
3028    ///             .name("book")
3029    ///             .await?;
3030    ///         Ok(receipt)
3031    ///     })
3032    ///     .name("reserve-and-book")
3033    ///     .retry_strategy(|_err, attempt| {
3034    ///         if attempt >= 3 {
3035    ///             RetryDecision::Stop
3036    ///         } else {
3037    ///             RetryDecision::retry(Duration::from_secs(5))
3038    ///         }
3039    ///     })
3040    ///     .await?;
3041    ///     Ok(result)
3042    /// }
3043    /// ```
3044    pub fn with_retry<O, F, Fut>(&self, f: F) -> WithRetryBuilder<O, F, Fut>
3045    where
3046        F: Fn(DurableContext, u32) -> Fut + Send + Sync + 'static,
3047        Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
3048        O: Send + 'static,
3049    {
3050        // The block claims its operation ID when the builder becomes its
3051        // future, because only then is its mode known.
3052        WithRetryBuilder::new_internal(self.clone(), f)
3053    }
3054
3055    /// Creates a wait-for-condition operation that polls until a predicate
3056    /// is satisfied.
3057    ///
3058    /// The check function is called repeatedly with the current state, and
3059    /// the configured strategy decides after each check whether to
3060    /// complete, keep polling, or fail. Set one with
3061    /// [`wait_strategy`](crate::builders::WaitForConditionBuilder::wait_strategy)
3062    /// (a bounded [`WaitStrategy`](crate::builders::wait_for_condition::WaitStrategy)
3063    /// configuration) or
3064    /// [`wait_strategy_fn`](crate::builders::WaitForConditionBuilder::wait_strategy_fn)
3065    /// (a custom closure). A strategy is required: the returned builder
3066    /// cannot be awaited, converted with `.future()` or started with
3067    /// `.spawn()` until one of the two setters has run.
3068    ///
3069    /// The first check receives the initial state after a round trip
3070    /// through the configured serdes, so it sees the same shape as a check
3071    /// that resumes from a recorded state. The round trip runs before the
3072    /// operation's first checkpoint: a serdes error marked with
3073    /// [`RetryableSerdesError`](crate::serdes::RetryableSerdesError) ends
3074    /// the invocation, and any other serdes error returns
3075    /// `WaitForCondition(SerializationFailed)` without calling the check.
3076    ///
3077    /// A check whose outcome the service has not recorded when the
3078    /// invocation ends, at the function timeout, in a crash or when a
3079    /// checkpoint write fails, runs again in the next invocation with the
3080    /// initial state, not with the state the previous check returned: the
3081    /// service clears the recorded state when it records the check's start.
3082    /// The re-run check keeps the attempt number of the check it replaces.
3083    ///
3084    /// A check error fails the operation without a retry. It records the
3085    /// type of a [`TypedError`](crate::TypedError) in its source chain, or
3086    /// `Error` when there is none. The error kind follows the recorded
3087    /// type, live and on replay: `WaitForCondition.MaxChecksExceeded` is
3088    /// reported as `MaxChecksExceeded`, `SerializationError` as
3089    /// `SerializationFailed`, and every other type as `CheckFailed`.
3090    ///
3091    /// # Examples
3092    ///
3093    /// ```no_run
3094    /// use aws_durable_execution_sdk as durable;
3095    /// use durable::builders::wait_for_condition::WaitDecision;
3096    /// use serde::{Deserialize, Serialize};
3097    /// use std::time::Duration;
3098    ///
3099    /// #[derive(Clone, Serialize, Deserialize)]
3100    /// struct PollState { count: u32 }
3101    ///
3102    /// async fn handler(
3103    ///     _event: serde_json::Value,
3104    ///     ctx: durable::DurableContext,
3105    /// ) -> Result<(), durable::BoxError> {
3106    ///     ctx.wait_for_condition(
3107    ///         |_step_ctx, state: PollState| async move {
3108    ///             Ok(PollState { count: state.count + 1 })
3109    ///         },
3110    ///         PollState { count: 0 },
3111    ///     ).name("poll-ready")
3112    ///      .wait_strategy_fn(|state: PollState, _attempt| {
3113    ///          if state.count >= 3 {
3114    ///              WaitDecision::complete()
3115    ///          } else {
3116    ///              WaitDecision::continue_with(Duration::from_secs(1))
3117    ///          }
3118    ///      })
3119    ///      .await?;
3120    ///     Ok(())
3121    /// }
3122    /// ```
3123    pub fn wait_for_condition<S, F, Fut>(
3124        &self,
3125        check: F,
3126        initial_state: S,
3127    ) -> WaitForConditionBuilder<S, F, Fut>
3128    where
3129        F: Fn(StepContext, S) -> Fut + Send + Sync + 'static,
3130        Fut: Future<Output = Result<S, BoxError>> + Send + 'static,
3131        S: Clone + Send + Sync + 'static,
3132    {
3133        let op_id = self.claim_operation_id("wait_for_condition");
3134        WaitForConditionBuilder::new_internal(self.clone(), op_id, initial_state, check)
3135    }
3136
3137    /// Creates a callback token for external completion.
3138    ///
3139    /// The returned [`Callback`](crate::builders::callback::Callback) provides an ID that
3140    /// external systems use to complete the operation, plus a
3141    /// [`DurableFuture`] that resolves when the callback arrives.
3142    ///
3143    /// # Examples
3144    ///
3145    /// ```no_run
3146    /// use aws_durable_execution_sdk as durable;
3147    /// use serde::{Deserialize, Serialize};
3148    ///
3149    /// #[derive(Serialize, Deserialize)]
3150    /// struct Approval { approved: bool }
3151    ///
3152    /// async fn handler(
3153    ///     _event: serde_json::Value,
3154    ///     ctx: durable::DurableContext,
3155    /// ) -> Result<bool, durable::BoxError> {
3156    ///     let cb = ctx.create_callback::<Approval>()
3157    ///         .name("approval")
3158    ///         .await?;
3159    ///     // Send cb.id() to an external system...
3160    ///     let approval = cb.result().await?;
3161    ///     Ok(approval.approved)
3162    /// }
3163    /// ```
3164    pub fn create_callback<O>(&self) -> CreateCallbackBuilder<O>
3165    where
3166        O: Send + 'static,
3167    {
3168        let op_id = self.claim_operation_id("create_callback");
3169        CreateCallbackBuilder::new_internal(self.clone(), op_id)
3170    }
3171
3172    /// Creates a wait-for-callback operation that registers and waits for
3173    /// an external callback in one step.
3174    ///
3175    /// The submitter closure receives the callback ID as an owned
3176    /// [`String`] and is responsible for delivering it to the external
3177    /// system that will complete it.
3178    ///
3179    /// The submitter is `FnOnce + Clone`. The SDK keeps the original closure
3180    /// and calls a fresh clone of it for each attempt. Each capture follows
3181    /// its type's `Clone` implementation: a cloned `Vec` or `String` has its
3182    /// own contents, and a cloned `Arc` shares its allocation.
3183    ///
3184    /// The output type `O` is the only type parameter: name it with
3185    /// `ctx.wait_for_callback::<Approval>(..)`, or let a typed binding or the
3186    /// function's return type set it. Every closure this method accepts
3187    /// implements [`Submitter`](crate::builders::Submitter).
3188    ///
3189    /// # Examples
3190    ///
3191    /// ```no_run
3192    /// use aws_durable_execution_sdk as durable;
3193    /// use serde::{Deserialize, Serialize};
3194    ///
3195    /// #[derive(Deserialize, Serialize)]
3196    /// struct Approval { ok: bool }
3197    ///
3198    /// async fn handler(
3199    ///     _event: serde_json::Value,
3200    ///     ctx: durable::DurableContext,
3201    /// ) -> Result<bool, durable::BoxError> {
3202    ///     let approval = ctx.wait_for_callback::<Approval>(
3203    ///         |_step_ctx, callback_id| async move {
3204    ///             // e.g., send callback_id to an approval queue
3205    ///             Ok(())
3206    ///         }
3207    ///     ).name("await-approval")
3208    ///      .await?;
3209    ///     Ok(approval.ok)
3210    /// }
3211    /// ```
3212    ///
3213    /// The returned builder's submitter type is opaque and borrows nothing,
3214    /// so a builder stored in a binding works after the context it came from
3215    /// is moved or dropped. The opaque type still names the `&self`
3216    /// lifetime, so a function that takes the context by value cannot return
3217    /// the un-awaited builder:
3218    ///
3219    /// ```compile_fail,E0597
3220    /// use aws_durable_execution_sdk as durable;
3221    /// use durable::builders::{Submitter, WaitForCallbackBuilder};
3222    ///
3223    /// fn approval(ctx: durable::DurableContext) -> WaitForCallbackBuilder<String, impl Submitter> {
3224    ///     ctx.wait_for_callback::<String>(|_step_ctx, _callback_id| async { Ok(()) })
3225    /// }
3226    /// ```
3227    ///
3228    /// Such a function returns `.future()` instead:
3229    ///
3230    /// ```no_run
3231    /// use aws_durable_execution_sdk as durable;
3232    ///
3233    /// fn approval(ctx: durable::DurableContext) -> durable::DurableFuture<String> {
3234    ///     ctx.wait_for_callback::<String>(|_step_ctx, _callback_id| async { Ok(()) })
3235    ///         .future()
3236    /// }
3237    /// ```
3238    pub fn wait_for_callback<O>(
3239        &self,
3240        submitter: impl Submitter,
3241    ) -> WaitForCallbackBuilder<O, impl Submitter>
3242    where
3243        O: Send + 'static,
3244    {
3245        let op_id = self.claim_operation_id("wait_for_callback");
3246        WaitForCallbackBuilder::new_internal(self.clone(), op_id, submitter)
3247    }
3248
3249    /// Creates a map operation that applies a function to each item in a
3250    /// collection, with configurable concurrency.
3251    ///
3252    /// Each item gets its own child context with an independent operation
3253    /// namespace.
3254    ///
3255    /// The closure should run at least one durable operation. Each item's
3256    /// result is saved, so a replay returns it without running the
3257    /// closure, with three exceptions. An item that returns an error is not
3258    /// retried, as a step is. An item whose result is over 256 KiB runs
3259    /// again on every replay, to rebuild the result. An item context that
3260    /// [`plugin_child_operations_depth`](crate::OptionsBuilder::plugin_child_operations_depth)
3261    /// preserves also runs again on every replay. So put plain computation
3262    /// in a step.
3263    ///
3264    /// When the whole batch result is over 256 KiB, the map records a
3265    /// summary of its outcome instead. With
3266    /// [`NestingMode::Normal`](crate::builders::map_parallel::NestingMode::Normal) replay reads each
3267    /// item from its own record. A
3268    /// [`NestingMode::Flat`](crate::builders::map_parallel::NestingMode::Flat) item has no record of
3269    /// its own, so replay runs the items again to rebuild the result, even
3270    /// when each item's result is small. [`MapBuilder::summary`] adds the
3271    /// caller's own summary to that record.
3272    ///
3273    /// The map claims one operation ID from this context when this method
3274    /// is called. Its items are numbered inside the map: item `i` has the
3275    /// ID of item `i` in the map's own namespace, whatever the item count,
3276    /// the concurrency or the order in which items finish. So this
3277    /// context's next operation gets this context's next ID.
3278    ///
3279    /// A panic in an item propagates to the caller with its original
3280    /// payload, and the SDK records nothing for it.
3281    ///
3282    /// # Examples
3283    ///
3284    /// ```no_run
3285    /// use aws_durable_execution_sdk as durable;
3286    /// use serde::{Deserialize, Serialize};
3287    ///
3288    /// #[derive(Clone, Serialize, Deserialize)]
3289    /// struct Image { url: String }
3290    ///
3291    /// #[derive(Serialize, Deserialize)]
3292    /// struct Thumbnail { url: String }
3293    ///
3294    /// async fn handler(
3295    ///     _event: serde_json::Value,
3296    ///     ctx: durable::DurableContext,
3297    /// ) -> Result<(), durable::BoxError> {
3298    ///     let images = vec![Image { url: "a.png".into() }, Image { url: "b.png".into() }];
3299    ///     let _thumbnails = ctx.map(images, |child_ctx, _img, _idx| async move {
3300    ///         let thumb = child_ctx
3301    ///             .step(|_| async { Ok(Thumbnail { url: "thumb.png".into() }) })
3302    ///             .await?;
3303    ///         Ok(thumb)
3304    ///     }).name("resize")
3305    ///       .max_concurrency(4)
3306    ///       .await?;
3307    ///     Ok(())
3308    /// }
3309    /// ```
3310    ///
3311    /// The closure gets one item. JS passes the whole input array as a
3312    /// fourth argument. To read other inputs, clone an `Arc<[I]>` of the
3313    /// inputs into the closure:
3314    ///
3315    /// ```no_run
3316    /// use std::sync::Arc;
3317    ///
3318    /// use aws_durable_execution_sdk as durable;
3319    ///
3320    /// async fn handler(
3321    ///     event: Vec<u32>,
3322    ///     ctx: durable::DurableContext,
3323    /// ) -> Result<Vec<u32>, durable::BoxError> {
3324    ///     let all: Arc<[u32]> = event.clone().into();
3325    ///     let shares = ctx
3326    ///         .map(event, move |child, item, _idx| {
3327    ///             let all = Arc::clone(&all);
3328    ///             async move {
3329    ///                 let share = child
3330    ///                     .step(move |_| async move {
3331    ///                         let total: u32 = all.iter().sum();
3332    ///                         Ok(item * 100 / total.max(1))
3333    ///                     })
3334    ///                     .name("share")
3335    ///                     .await?;
3336    ///                 Ok(share)
3337    ///             }
3338    ///         })
3339    ///         .name("shares")
3340    ///         .await?;
3341    ///     Ok(shares)
3342    /// }
3343    /// ```
3344    pub fn map<Items, I, O, F, Fut>(&self, items: Items, f: F) -> MapBuilder<I, O, F, Fut>
3345    where
3346        Items: IntoIterator<Item = I>,
3347        F: Fn(DurableContext, I, usize) -> Fut + Send + Sync + 'static,
3348        Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
3349        I: Send + 'static,
3350        O: Send + 'static,
3351    {
3352        let op_id = self.claim_operation_id("map");
3353        // The closure is stored unerased; at execution it is shared as
3354        // `Arc<F>` and every item produces a concrete future, so the
3355        // internal JoinSet holds unboxed futures. The single erasure point
3356        // is the builder's `.future()` / `.await`, producing one
3357        // DurableFuture box.
3358        let items: Vec<I> = items.into_iter().collect();
3359        MapBuilder::new_internal(self.clone(), op_id, items, f)
3360    }
3361
3362    /// Creates a parallel operation that executes named branches
3363    /// concurrently.
3364    ///
3365    /// The parallel claims one operation ID from this context when this
3366    /// method is called. Its branches are numbered inside the parallel:
3367    /// branch `i` has the ID of item `i` in the parallel's own namespace,
3368    /// as a [`map`](Self::map) item does.
3369    ///
3370    /// A panic in a branch propagates to the caller with its original
3371    /// payload, and the SDK records nothing for it.
3372    ///
3373    /// Each branch should run at least one durable operation. Each branch's
3374    /// result is saved, so a replay returns it without running the branch,
3375    /// with three exceptions. A branch that returns an error is not
3376    /// retried, as a step is. A branch whose result is over 256 KiB runs
3377    /// again on every replay, to rebuild the result. A branch context that
3378    /// [`plugin_child_operations_depth`](crate::OptionsBuilder::plugin_child_operations_depth)
3379    /// preserves also runs again on every replay. So put plain computation
3380    /// in a step.
3381    ///
3382    /// When the whole batch result is over 256 KiB, the parallel records a
3383    /// summary of its outcome instead. With
3384    /// [`NestingMode::Normal`](crate::builders::map_parallel::NestingMode::Normal) replay reads each
3385    /// branch from its own record. A
3386    /// [`NestingMode::Flat`](crate::builders::map_parallel::NestingMode::Flat) branch has no record of
3387    /// its own, so replay runs the branches again to rebuild the result, even
3388    /// when each branch's result is small. [`ParallelBuilder::summary`] adds the
3389    /// caller's own summary to that record.
3390    ///
3391    /// # Examples
3392    ///
3393    /// ```no_run
3394    /// use aws_durable_execution_sdk as durable;
3395    ///
3396    /// async fn handler(
3397    ///     _event: serde_json::Value,
3398    ///     ctx: durable::DurableContext,
3399    /// ) -> Result<(), durable::BoxError> {
3400    ///     let branches = vec![
3401    ///         durable::Branch::new("a", |child_ctx: durable::DurableContext| async move {
3402    ///             let v = child_ctx.step(|_| async { Ok(1) }).await?;
3403    ///             Ok(v)
3404    ///         }),
3405    ///     ];
3406    ///     let _results: Vec<i32> = ctx.parallel(branches)
3407    ///         .name("fan-out")
3408    ///         .await?;
3409    ///     Ok(())
3410    /// }
3411    /// ```
3412    pub fn parallel<Branches, O>(&self, branches: Branches) -> ParallelBuilder<O>
3413    where
3414        Branches: IntoIterator<Item = Branch<O>>,
3415        O: Send + 'static,
3416    {
3417        let op_id = self.claim_operation_id("parallel");
3418        let branch_tuples: Vec<_> = branches.into_iter().map(Branch::into_parts).collect();
3419        ParallelBuilder::new_internal(self.clone(), op_id, branch_tuples)
3420    }
3421
3422    /// Creates a parallel operation over branches whose results have
3423    /// different types.
3424    ///
3425    /// Each branch of `branches` returns its own type, and the
3426    /// [`BranchHandle`](crate::BranchHandle) that
3427    /// [`BranchSet::add`](crate::BranchSet::add) or
3428    /// [`BranchSet::add_with_serdes`](crate::BranchSet::add_with_serdes)
3429    /// returned gives that type back. The operation records the same
3430    /// history as [`parallel`](Self::parallel) over branches with the same
3431    /// names and outcomes: one parallel parent and one child context per
3432    /// branch. Only the encoded branch values differ, because each branch
3433    /// encodes its value with its own serdes.
3434    ///
3435    /// `.await` returns the [`CompletionReason`](crate::builders::map_parallel::CompletionReason)
3436    /// of the batch. It returns an error in exactly the cases where
3437    /// `parallel(...).await` does, with the same error. When it returns,
3438    /// each handle whose branch settled holds the branch's value or error.
3439    /// A handle whose branch never started, or still ran when the batch
3440    /// completed, holds no result. This holds whatever the parallel's own
3441    /// outcome, including an error from a completion rule that fails the
3442    /// batch. When the batch does not complete, for example because a
3443    /// checkpoint write fails, no handle holds a result. With
3444    /// [`spawn`](crate::builders::ParallelMixedBuilder::spawn), the handles
3445    /// are filled when the returned future is awaited.
3446    ///
3447    /// The operation ID is claimed when this method is called, as
3448    /// [`parallel`](Self::parallel) claims it, and branch `i` has the ID of
3449    /// item `i` in the batch's own namespace, as in `parallel`.
3450    ///
3451    /// # Which form to use
3452    ///
3453    /// - [`parallel`](Self::parallel), when every branch returns the same
3454    ///   type.
3455    /// - [`parallel`](Self::parallel) over an enum, when the results are
3456    ///   cases of one type that the handler processes together. Use serde's
3457    ///   default (externally tagged) enum representation.
3458    ///   `#[serde(untagged)]` can decode a result into the wrong variant
3459    ///   when two variants accept the same JSON.
3460    /// - `parallel_mixed`, when each branch returns its own type.
3461    ///
3462    /// A panic in a branch propagates to the caller with its original
3463    /// payload, the SDK records nothing for it, and no handle receives a
3464    /// result.
3465    ///
3466    /// # Examples
3467    ///
3468    /// ```no_run
3469    /// use aws_durable_execution_sdk as durable;
3470    /// use durable::serdes::SerdesContext;
3471    /// use durable::{BoxError, BranchSet, Serdes};
3472    ///
3473    /// #[derive(serde::Serialize, serde::Deserialize)]
3474    /// struct Reservation {
3475    ///     sku: String,
3476    /// }
3477    ///
3478    /// struct PaymentAuth {
3479    ///     code: u32,
3480    /// }
3481    ///
3482    /// // A serdes for `PaymentAuth` only.
3483    /// struct PaymentSerdes;
3484    ///
3485    /// impl Serdes<PaymentAuth> for PaymentSerdes {
3486    ///     async fn serialize(&self, value: PaymentAuth, _ctx: SerdesContext) -> Result<String, BoxError> {
3487    ///         Ok(value.code.to_string())
3488    ///     }
3489    ///     async fn deserialize(&self, wire: String, _ctx: SerdesContext) -> Result<PaymentAuth, BoxError> {
3490    ///         Ok(PaymentAuth { code: wire.parse()? })
3491    ///     }
3492    /// }
3493    ///
3494    /// async fn handler(
3495    ///     _event: serde_json::Value,
3496    ///     ctx: durable::DurableContext,
3497    /// ) -> Result<String, BoxError> {
3498    ///     let mut branches = BranchSet::new();
3499    ///     let inventory = branches.add("inventory", |_ctx| async {
3500    ///         Ok(Reservation { sku: "sku-1".to_owned() })
3501    ///     });
3502    ///     let payment = branches.add_with_serdes("payment", PaymentSerdes, |_ctx| async {
3503    ///         Ok(PaymentAuth { code: 7 })
3504    ///     });
3505    ///
3506    ///     ctx.parallel_mixed(branches)
3507    ///         .name("process-order")
3508    ///         .max_concurrency(2)
3509    ///         .await?;
3510    ///
3511    ///     let inventory: Reservation = inventory.into_result()?;
3512    ///     let payment: PaymentAuth = payment.into_result()?;
3513    ///     Ok(format!("{}: {}", payment.code, inventory.sku))
3514    /// }
3515    /// ```
3516    pub fn parallel_mixed(&self, branches: crate::BranchSet) -> ParallelMixedBuilder {
3517        let op_id = self.claim_operation_id("parallel_mixed");
3518        ParallelMixedBuilder::new_internal(self.clone(), op_id, branches)
3519    }
3520
3521    /// Joins all futures, failing fast on the first error.
3522    ///
3523    /// Returns `Vec<O>` on success, or the first `OperationError`
3524    /// encountered.
3525    ///
3526    /// The returned value has passed through the combinator's JSON
3527    /// encoding: the SDK decodes the payload it recorded and returns that.
3528    /// So the first run returns the value a replay returns, and data that
3529    /// the value's serde form omits, such as a `#[serde(skip)]` field, is
3530    /// absent on both paths.
3531    ///
3532    /// When the combinator resolves, it cancels every input it started from
3533    /// `.future()` that is still running, and a cancelled input stops at its
3534    /// next `.await`. An input started with `.spawn()` keeps running, and its
3535    /// result is dropped.
3536    ///
3537    /// # Oversized results
3538    ///
3539    /// A combined result larger than 256 KiB is not stored in the
3540    /// combinator's record. Replay then drives the inputs again. An input
3541    /// whose own terminal record is inline replays without running again. An
3542    /// input whose own terminal record has `ReplayChildren` runs its body
3543    /// again to rebuild its value, and its recorded descendants replay.
3544    ///
3545    /// # Empty input
3546    ///
3547    /// Called with no futures, resolves to `Ok` with an empty `Vec`,
3548    /// matching `futures::future::try_join_all`.
3549    ///
3550    /// A panic in an input propagates to the caller with its original
3551    /// payload, and the SDK records nothing for it.
3552    ///
3553    /// # Operation ID
3554    ///
3555    /// This method collects `futures` and claims no operation ID. An
3556    /// operation that the iterator creates claims its ID during this
3557    /// collection, so the inputs take their IDs before the combinator. The
3558    /// combinator claims its own ID when it is first awaited or polled, or
3559    /// when it is spawned, on this context. A combinator that is dropped
3560    /// before that claims no ID and records nothing. An input that is itself
3561    /// a combinator must already have claimed its ID: start it with
3562    /// `.spawn()` and pass the returned handle. A still-unclaimed combinator
3563    /// input fails the execution with a context usage error. Starting the
3564    /// combinator inside the body of another context (a child context, a
3565    /// `with_retry` attempt, a map item or a parallel branch) fails the
3566    /// execution with a context usage error too.
3567    ///
3568    /// # Start order
3569    ///
3570    /// When the combinator runs, it begins the start phase of each
3571    /// `.future()` input, in input order, before its own start step, as a
3572    /// JS combinator awaits inputs that already run. An input whose JS form
3573    /// queues its `START` at the call (a step, a wait, a child context,
3574    /// `with_retry`, a map, a parallel or a `wait_for_callback`) queues it
3575    /// then, so its first checkpoint comes before the combinator's, and a
3576    /// start hook it fires when it queues `START` runs before the
3577    /// combinator's. A start hook that waits for the `START` response (a
3578    /// wait, a step under `AtMostOncePerRetry`) keeps that timing. A lazy
3579    /// invoke or `wait_for_condition` is polled once, in input order,
3580    /// before the combinator's start step, but the combinator does not wait
3581    /// for its `START`. The combinator then queues its own `START`, fires
3582    /// its start hook, and runs the inputs without waiting for the `START`
3583    /// response; its `SUCCEED` or `FAIL` waits for it. On a `STARTED`,
3584    /// `PENDING` or `READY` record it fires its replay start hook after the
3585    /// inputs' start phases begin. A replay of a final record starts no
3586    /// `.future()` input. A handle that `.spawn()` returned already runs
3587    /// independently, and the combinator does not start it again.
3588    ///
3589    /// # Examples
3590    ///
3591    /// ```no_run
3592    /// use aws_durable_execution_sdk as durable;
3593    ///
3594    /// async fn handler(
3595    ///     _event: serde_json::Value,
3596    ///     ctx: durable::DurableContext,
3597    /// ) -> Result<(), durable::BoxError> {
3598    ///     let a = ctx.step(|_| async { Ok(1) }).future();
3599    ///     let b = ctx.step(|_| async { Ok(2) }).future();
3600    ///     let _results: Vec<i32> = ctx.try_join_all([a, b])
3601    ///         .name("gather")
3602    ///         .await?;
3603    ///     Ok(())
3604    /// }
3605    /// ```
3606    pub fn try_join_all<O>(
3607        &self,
3608        futures: impl IntoIterator<Item = DurableFuture<O>>,
3609    ) -> TryJoinAllBuilder<O>
3610    where
3611        O: Serialize + DeserializeOwned + Send + 'static,
3612    {
3613        TryJoinAllBuilder::new_internal(self.clone(), futures.into_iter().collect())
3614    }
3615
3616    /// Joins all futures, collecting every outcome as a `Result`.
3617    ///
3618    /// Never fails fast: all futures run to completion regardless of
3619    /// individual errors. The output holds one `Result<O, OperationError>`
3620    /// per input, in input order.
3621    ///
3622    /// # Outer and inner errors
3623    ///
3624    /// Awaiting returns `Result<Vec<Result<O, OperationError>>, OperationError>`,
3625    /// and the two levels report different failures:
3626    ///
3627    /// 1. The outer `Err` means the combinator itself failed, for example
3628    ///    replay nondeterminism. A failed checkpoint write never reaches
3629    ///    it: the write fails the invocation, and the combinator does not
3630    ///    resolve.
3631    /// 2. An input that returns `Err` appears only as that input's inner
3632    ///    `Err`. The outer result is then `Ok`.
3633    /// 3. A panicking input ends the invocation, as a panic does everywhere
3634    ///    else in the SDK. The SDK resumes the original panic payload. The
3635    ///    combinator writes no record and returns no result.
3636    /// 4. Nothing cancels an input that `join_all` awaits. A combinator stops
3637    ///    an input only by dropping it, and `join_all` drops no input before
3638    ///    it settles.
3639    ///
3640    /// An inner `Err` is a
3641    /// [`OperationErrorKind::ChildContext`](crate::OperationErrorKind::ChildContext)
3642    /// error of kind
3643    /// [`ChildContextErrorKind::ChildFailed`](crate::ChildContextErrorKind::ChildFailed),
3644    /// whose `wire()` carries the input's recorded failure (`error_type`,
3645    /// message, `error_data`), on the first run and on replay. Match on the
3646    /// `err.wire()` fields, not on the kind of the input that failed.
3647    ///
3648    /// # Oversized results
3649    ///
3650    /// A combined result larger than 256 KiB is not stored in the
3651    /// combinator's record. Replay then drives the inputs again. An input
3652    /// whose own terminal record is inline replays without running again. An
3653    /// input whose own terminal record has `ReplayChildren` runs its body
3654    /// again to rebuild its value, and its recorded descendants replay.
3655    ///
3656    /// # Empty input
3657    ///
3658    /// Called with no futures, resolves to `Ok` with an empty `Vec`,
3659    /// matching `futures::future::join_all`.
3660    ///
3661    /// # Operation ID
3662    ///
3663    /// This method collects `futures` and claims no operation ID. An
3664    /// operation that the iterator creates claims its ID during this
3665    /// collection, so the inputs take their IDs before the combinator. The
3666    /// combinator claims its own ID when it is first awaited or polled, or
3667    /// when it is spawned, on this context. A combinator that is dropped
3668    /// before that claims no ID and records nothing. An input that is itself
3669    /// a combinator must already have claimed its ID: start it with
3670    /// `.spawn()` and pass the returned handle. A still-unclaimed combinator
3671    /// input fails the execution with a context usage error. Starting the
3672    /// combinator inside the body of another context (a child context, a
3673    /// `with_retry` attempt, a map item or a parallel branch) fails the
3674    /// execution with a context usage error too.
3675    ///
3676    /// # Start order
3677    ///
3678    /// When the combinator runs, it begins the start phase of each
3679    /// `.future()` input, in input order, before its own start step, as a
3680    /// JS combinator awaits inputs that already run. An input whose JS form
3681    /// queues its `START` at the call (a step, a wait, a child context,
3682    /// `with_retry`, a map, a parallel or a `wait_for_callback`) queues it
3683    /// then, so its first checkpoint comes before the combinator's, and a
3684    /// start hook it fires when it queues `START` runs before the
3685    /// combinator's. A start hook that waits for the `START` response (a
3686    /// wait, a step under `AtMostOncePerRetry`) keeps that timing. A lazy
3687    /// invoke or `wait_for_condition` is polled once, in input order,
3688    /// before the combinator's start step, but the combinator does not wait
3689    /// for its `START`. The combinator then queues its own `START`, fires
3690    /// its start hook, and runs the inputs without waiting for the `START`
3691    /// response; its `SUCCEED` or `FAIL` waits for it. On a `STARTED`,
3692    /// `PENDING` or `READY` record it fires its replay start hook after the
3693    /// inputs' start phases begin. A replay of a final record starts no
3694    /// `.future()` input. A handle that `.spawn()` returned already runs
3695    /// independently, and the combinator does not start it again.
3696    ///
3697    /// # Examples
3698    ///
3699    /// ```no_run
3700    /// use aws_durable_execution_sdk as durable;
3701    ///
3702    /// async fn handler(
3703    ///     _event: serde_json::Value,
3704    ///     ctx: durable::DurableContext,
3705    /// ) -> Result<(), durable::BoxError> {
3706    ///     let a = ctx.step(|_| async { Ok(1) }).future();
3707    ///     let b = ctx.step(|_| async { Ok(2) }).future();
3708    ///     let outcomes: Vec<Result<i32, durable::OperationError>> = ctx
3709    ///         .join_all([a, b])
3710    ///         .name("collect")
3711    ///         .await?;
3712    ///     let succeeded = outcomes.iter().filter(|outcome| outcome.is_ok()).count();
3713    ///     assert_eq!(succeeded, 2);
3714    ///     Ok(())
3715    /// }
3716    /// ```
3717    pub fn join_all<O>(
3718        &self,
3719        futures: impl IntoIterator<Item = DurableFuture<O>>,
3720    ) -> JoinAllBuilder<O>
3721    where
3722        O: Serialize + DeserializeOwned + Send + 'static,
3723    {
3724        JoinAllBuilder::new_internal(self.clone(), futures.into_iter().collect())
3725    }
3726
3727    /// Returns the first successful result.
3728    ///
3729    /// When the combinator resolves, it cancels every input it started from
3730    /// `.future()` that is still running, and a cancelled input stops at its
3731    /// next `.await`. An input started with `.spawn()` keeps running, and its
3732    /// result is dropped.
3733    ///
3734    /// The returned value has passed through the combinator's JSON
3735    /// encoding: the SDK decodes the payload it recorded and returns that.
3736    /// So the first run returns the value a replay returns, and data that
3737    /// the value's serde form omits, such as a `#[serde(skip)]` field, is
3738    /// absent on both paths.
3739    ///
3740    /// If every future fails, the operation fails with
3741    /// [`CombinatorErrorKind::AllFailed`](crate::CombinatorErrorKind::AllFailed)
3742    /// preserving each future's error (see [`CombinatorError::failures`](crate::CombinatorError::failures)).
3743    ///
3744    /// # Oversized results
3745    ///
3746    /// A winning value larger than 256 KiB is not stored in the
3747    /// combinator's record. The record stores the winner's input index
3748    /// instead. Replay then drives only the recorded winner again. It drops
3749    /// every other `.future()` input without running it, and an input
3750    /// started with `.spawn()` keeps running. The winner replays without
3751    /// running again when its own terminal record is inline. A winner whose
3752    /// own terminal record has `ReplayChildren` runs its body again to
3753    /// rebuild its value, and its recorded descendants replay.
3754    ///
3755    /// # Empty input
3756    ///
3757    /// Called with no futures, fails with
3758    /// [`CombinatorErrorKind::EmptyInput`](crate::CombinatorErrorKind::EmptyInput):
3759    /// there is no future that could succeed.
3760    ///
3761    /// A panic in an input propagates to the caller with its original
3762    /// payload, and the SDK records nothing for it.
3763    ///
3764    /// # Operation ID
3765    ///
3766    /// This method collects `futures` and claims no operation ID. An
3767    /// operation that the iterator creates claims its ID during this
3768    /// collection, so the inputs take their IDs before the combinator. The
3769    /// combinator claims its own ID when it is first awaited or polled, or
3770    /// when it is spawned, on this context. A combinator that is dropped
3771    /// before that claims no ID and records nothing. An input that is itself
3772    /// a combinator must already have claimed its ID: start it with
3773    /// `.spawn()` and pass the returned handle. A still-unclaimed combinator
3774    /// input fails the execution with a context usage error. Starting the
3775    /// combinator inside the body of another context (a child context, a
3776    /// `with_retry` attempt, a map item or a parallel branch) fails the
3777    /// execution with a context usage error too.
3778    ///
3779    /// # Start order
3780    ///
3781    /// When the combinator runs, it begins the start phase of each
3782    /// `.future()` input, in input order, before its own start step, as a
3783    /// JS combinator awaits inputs that already run. An input whose JS form
3784    /// queues its `START` at the call (a step, a wait, a child context,
3785    /// `with_retry`, a map, a parallel or a `wait_for_callback`) queues it
3786    /// then, so its first checkpoint comes before the combinator's, and a
3787    /// start hook it fires when it queues `START` runs before the
3788    /// combinator's. A start hook that waits for the `START` response (a
3789    /// wait, a step under `AtMostOncePerRetry`) keeps that timing. A lazy
3790    /// invoke or `wait_for_condition` is polled once, in input order,
3791    /// before the combinator's start step, but the combinator does not wait
3792    /// for its `START`. The combinator then queues its own `START`, fires
3793    /// its start hook, and runs the inputs without waiting for the `START`
3794    /// response; its `SUCCEED` or `FAIL` waits for it. On a `STARTED`,
3795    /// `PENDING` or `READY` record it fires its replay start hook after the
3796    /// inputs' start phases begin. A replay of a final record starts no
3797    /// `.future()` input. A handle that `.spawn()` returned already runs
3798    /// independently, and the combinator does not start it again.
3799    ///
3800    /// # Examples
3801    ///
3802    /// ```no_run
3803    /// use aws_durable_execution_sdk as durable;
3804    ///
3805    /// async fn handler(
3806    ///     _event: serde_json::Value,
3807    ///     ctx: durable::DurableContext,
3808    /// ) -> Result<String, durable::BoxError> {
3809    ///     let a = ctx.step(|_| async { Ok("fast".to_owned()) }).future();
3810    ///     let b = ctx.step(|_| async { Ok("slow".to_owned()) }).future();
3811    ///     let winner = ctx.select_ok([a, b])
3812    ///         .name("race-ok")
3813    ///         .await?;
3814    ///     Ok(winner)
3815    /// }
3816    /// ```
3817    pub fn select_ok<O>(
3818        &self,
3819        futures: impl IntoIterator<Item = DurableFuture<O>>,
3820    ) -> SelectOkBuilder<O>
3821    where
3822        O: Serialize + DeserializeOwned + Send + 'static,
3823    {
3824        SelectOkBuilder::new_internal(self.clone(), futures.into_iter().collect())
3825    }
3826
3827    /// Returns the first settled result, whether success or failure.
3828    ///
3829    /// When the combinator resolves, it cancels every input it started from
3830    /// `.future()` that is still running, and a cancelled input stops at its
3831    /// next `.await`. An input started with `.spawn()` keeps running, and its
3832    /// result is dropped.
3833    ///
3834    /// The returned value has passed through the combinator's JSON
3835    /// encoding: the SDK decodes the payload it recorded and returns that.
3836    /// So the first run returns the value a replay returns, and data that
3837    /// the value's serde form omits, such as a `#[serde(skip)]` field, is
3838    /// absent on both paths.
3839    ///
3840    /// When the first settled outcome is a failure, the operation fails
3841    /// with
3842    /// [`CombinatorErrorKind::FirstSettledFailed`](crate::CombinatorErrorKind::FirstSettledFailed)
3843    /// carrying the losing future's error as its source; the same variant
3844    /// is produced live and on replay.
3845    ///
3846    /// # Oversized results
3847    ///
3848    /// A winning value larger than 256 KiB is not stored in the
3849    /// combinator's record. The record stores the winner's input index
3850    /// instead. Replay then drives only the recorded winner again. It drops
3851    /// every other `.future()` input without running it, and an input
3852    /// started with `.spawn()` keeps running. The winner replays without
3853    /// running again when its own terminal record is inline. A winner whose
3854    /// own terminal record has `ReplayChildren` runs its body again to
3855    /// rebuild its value, and its recorded descendants replay.
3856    ///
3857    /// # Empty input
3858    ///
3859    /// Called with no futures, fails with
3860    /// [`CombinatorErrorKind::EmptyInput`](crate::CombinatorErrorKind::EmptyInput):
3861    /// there is no future that could settle.
3862    ///
3863    /// A panic in an input propagates to the caller with its original
3864    /// payload, and the SDK records nothing for it.
3865    ///
3866    /// # Operation ID
3867    ///
3868    /// This method collects `futures` and claims no operation ID. An
3869    /// operation that the iterator creates claims its ID during this
3870    /// collection, so the inputs take their IDs before the combinator. The
3871    /// combinator claims its own ID when it is first awaited or polled, or
3872    /// when it is spawned, on this context. A combinator that is dropped
3873    /// before that claims no ID and records nothing. An input that is itself
3874    /// a combinator must already have claimed its ID: start it with
3875    /// `.spawn()` and pass the returned handle. A still-unclaimed combinator
3876    /// input fails the execution with a context usage error. Starting the
3877    /// combinator inside the body of another context (a child context, a
3878    /// `with_retry` attempt, a map item or a parallel branch) fails the
3879    /// execution with a context usage error too.
3880    ///
3881    /// # Start order
3882    ///
3883    /// When the combinator runs, it begins the start phase of each
3884    /// `.future()` input, in input order, before its own start step, as a
3885    /// JS combinator awaits inputs that already run. An input whose JS form
3886    /// queues its `START` at the call (a step, a wait, a child context,
3887    /// `with_retry`, a map, a parallel or a `wait_for_callback`) queues it
3888    /// then, so its first checkpoint comes before the combinator's, and a
3889    /// start hook it fires when it queues `START` runs before the
3890    /// combinator's. A start hook that waits for the `START` response (a
3891    /// wait, a step under `AtMostOncePerRetry`) keeps that timing. A lazy
3892    /// invoke or `wait_for_condition` is polled once, in input order,
3893    /// before the combinator's start step, but the combinator does not wait
3894    /// for its `START`. The combinator then queues its own `START`, fires
3895    /// its start hook, and runs the inputs without waiting for the `START`
3896    /// response; its `SUCCEED` or `FAIL` waits for it. On a `STARTED`,
3897    /// `PENDING` or `READY` record it fires its replay start hook after the
3898    /// inputs' start phases begin. A replay of a final record starts no
3899    /// `.future()` input. A handle that `.spawn()` returned already runs
3900    /// independently, and the combinator does not start it again.
3901    ///
3902    /// # Examples
3903    ///
3904    /// ```no_run
3905    /// use aws_durable_execution_sdk as durable;
3906    ///
3907    /// async fn handler(
3908    ///     _event: serde_json::Value,
3909    ///     ctx: durable::DurableContext,
3910    /// ) -> Result<String, durable::BoxError> {
3911    ///     let a = ctx.step(|_| async { Ok("first".to_owned()) }).future();
3912    ///     let b = ctx.step(|_| async { Ok("second".to_owned()) }).future();
3913    ///     let winner = ctx.race([a, b])
3914    ///         .name("fastest")
3915    ///         .await?;
3916    ///     Ok(winner)
3917    /// }
3918    /// ```
3919    pub fn race<O>(&self, futures: impl IntoIterator<Item = DurableFuture<O>>) -> RaceBuilder<O>
3920    where
3921        O: Serialize + DeserializeOwned + Send + 'static,
3922    {
3923        RaceBuilder::new_internal(self.clone(), futures.into_iter().collect())
3924    }
3925}
3926
3927/// Context passed to step bodies.
3928///
3929/// `StepContext` deliberately does **not** expose any durable operations.
3930/// This provides compile-time enforcement of the rule that durable
3931/// operations cannot be nested inside step bodies: attempting to call
3932/// a durable operation method inside a step body is a type error.
3933///
3934/// Use `tracing` macros for logging inside steps: the SDK-created
3935/// operation span automatically attaches execution context fields.
3936///
3937/// # Examples
3938///
3939/// ```no_run
3940/// use aws_durable_execution_sdk as durable;
3941///
3942/// async fn handler(
3943///     _event: serde_json::Value,
3944///     ctx: durable::DurableContext,
3945/// ) -> Result<String, durable::BoxError> {
3946///     let result = ctx.step(|step_ctx: durable::StepContext| async move {
3947///         // step_ctx has no durable operations: type-system enforced
3948///         tracing::info!("inside step");
3949///         Ok("done".to_owned())
3950///     }).await?;
3951///     Ok(result)
3952/// }
3953/// ```
3954#[derive(Debug, Clone)]
3955#[non_exhaustive]
3956pub struct StepContext {
3957    attempt: u32,
3958    _private: (),
3959}
3960
3961impl StepContext {
3962    /// Creates a new step context (internal).
3963    pub(crate) fn new(attempt: u32) -> Self {
3964        Self {
3965            attempt,
3966            _private: (),
3967        }
3968    }
3969
3970    /// Returns the current attempt number, starting at 1 for the first
3971    /// attempt.
3972    pub fn attempt(&self) -> u32 {
3973        self.attempt
3974    }
3975}
3976
3977/// Canonicalizes an operation type string to the service wire format via
3978/// [`aws_sdk_lambda::types::OperationType`].
3979///
3980/// Both envelope parsers store the wire token (`CHAINED_INVOKE`, the
3981/// `OperationType::as_str()` form). The SDK's own claimed types and older
3982/// fixtures use `PascalCase` (`ChainedInvoke`). A plain case-insensitive
3983/// comparison misses the underscore, so both spellings are converted to
3984/// `SCREAMING_SNAKE` and round-tripped through `OperationType` before
3985/// comparison. Unknown types canonicalize to their `SCREAMING_SNAKE` form,
3986/// which keeps genuine mismatches detectable.
3987pub(crate) fn canonical_op_type(raw: &str) -> String {
3988    // PascalCase → SCREAMING_SNAKE: insert `_` at lower→upper boundaries,
3989    // then uppercase. Already-wire values (all caps + underscores) pass
3990    // through unchanged because they contain no lower→upper boundary.
3991    let mut screaming = String::with_capacity(raw.len() + 4);
3992    let mut prev_is_lower = false;
3993    for ch in raw.chars() {
3994        if ch.is_ascii_uppercase() && prev_is_lower {
3995            screaming.push('_');
3996        }
3997        prev_is_lower = ch.is_ascii_lowercase();
3998        screaming.push(ch.to_ascii_uppercase());
3999    }
4000    // Round-trip through the SDK enum so every known type lands on the
4001    // exact canonical wire constant (`OperationType::as_str()`).
4002    aws_sdk_lambda::types::OperationType::from(screaming.as_str())
4003        .as_str()
4004        .to_owned()
4005}
4006
4007/// Compares a checkpoint record's stored identity against a claimed one.
4008///
4009/// Returns `None` when the identities match (or the record carries no
4010/// operation type), and `Some(expected)`, the stored identity
4011/// formatted for the mismatch error, when they differ. The comparisons
4012/// borrow the record, so the match path clones nothing; only the cold
4013/// mismatch path allocates. This is the shared core behind
4014/// `DurableContext::validate_replay_identity` and
4015/// `DurableContext::checkpoint_view_validated`.
4016fn replay_identity_mismatch(
4017    record: &CheckpointRecord,
4018    claimed_type: &str,
4019    claimed_sub_type: Option<&str>,
4020    claimed_name: Option<&str>,
4021) -> Option<String> {
4022    // A record without a stored operation type has nothing to validate
4023    // against, so the check skips it, as JS does. This is the ONLY
4024    // lenient path; once a record carries identity, every field is
4025    // compared in full.
4026    let expected_type = record.op_type.as_deref()?;
4027
4028    let mismatch = || {
4029        Some(format_op_identity(
4030            expected_type,
4031            record.sub_type.as_deref(),
4032            record.op_name.as_deref(),
4033        ))
4034    };
4035
4036    // Compare canonicalized types: the checkpoint log stores the wire
4037    // token (e.g. `CHAINED_INVOKE`), while the claimed type can be
4038    // PascalCase, so both sides canonicalize through `OperationType`
4039    // before comparison.
4040    if canonical_op_type(expected_type) != canonical_op_type(claimed_type) {
4041        return mismatch();
4042    }
4043
4044    // Sub-type is compared as a complete Option in both directions: a
4045    // stored sub-type with no claimed sub-type (or the reverse) is a
4046    // mismatch, not a skip. Otherwise a reordered same-type operation
4047    // could consume the wrong checkpoint silently.
4048    let sub_type_matches = match (record.sub_type.as_deref(), claimed_sub_type) {
4049        (Some(expected), Some(claimed)) => expected.eq_ignore_ascii_case(claimed),
4050        (None, None) => true,
4051        (Some(_), None) | (None, Some(_)) => false,
4052    };
4053    if !sub_type_matches {
4054        return mismatch();
4055    }
4056
4057    // Name is likewise compared as a complete Option in both directions.
4058    // Adding or removing `.name(...)` between runs changes the operation's
4059    // replay identity and must be flagged, for the same reason as above.
4060    //
4061    // Empty names are normalized to `None` on BOTH sides before the
4062    // comparison: an empty name and no name are the same identity, so an
4063    // operation named `""` whose record holds no `Name` still matches.
4064    // Without normalization an UNCHANGED handler would be rejected on
4065    // resume. Map items and parallel branches never claim an empty name:
4066    // they claim the recorded name, which defaults to `map-item-{index}`
4067    // or `parallel-branch-{index}`.
4068    let claimed_name = claimed_name.filter(|n| !n.is_empty());
4069    let expected_name = record.op_name.as_deref().filter(|n| !n.is_empty());
4070    let name_matches = match (expected_name, claimed_name) {
4071        (Some(expected), Some(claimed)) => expected == claimed,
4072        (None, None) => true,
4073        (Some(_), None) | (None, Some(_)) => false,
4074    };
4075    if !name_matches {
4076        return mismatch();
4077    }
4078
4079    None
4080}
4081
4082/// Formats an operation's identity as a human-readable string for error
4083/// messages (e.g. `"Step/Step named \"fetch-name\""` or `"Wait/Wait"`).
4084fn format_op_identity(op_type: &str, sub_type: Option<&str>, name: Option<&str>) -> String {
4085    let mut s = String::with_capacity(64);
4086    s.push_str(op_type);
4087    if let Some(sub) = sub_type {
4088        s.push('/');
4089        s.push_str(sub);
4090    }
4091    if let Some(n) = name {
4092        s.push_str(" named \"");
4093        s.push_str(n);
4094        s.push('"');
4095    }
4096    s
4097}
4098
4099/// The value an operation returns when its checkpoint write found
4100/// no Tokio runtime (see `DurableContext::checkpoint_failure_unrecoverable`).
4101pub(crate) trait NoRuntimeOutcome {
4102    /// The outcome for `no_runtime`.
4103    fn no_runtime(no_runtime: crate::task::NoRuntime) -> Self;
4104}
4105
4106impl<T> NoRuntimeOutcome for Result<T, OperationError> {
4107    fn no_runtime(no_runtime: crate::task::NoRuntime) -> Self {
4108        Err(crate::task::NoRuntime::operation_error(no_runtime))
4109    }
4110}
4111
4112/// For a call site whose function returns the operation error itself.
4113impl NoRuntimeOutcome for OperationError {
4114    fn no_runtime(no_runtime: crate::task::NoRuntime) -> Self {
4115        crate::task::NoRuntime::operation_error(no_runtime)
4116    }
4117}
4118
4119#[cfg(test)]
4120// Tests deliberately spawn foreign (unblessed) tasks to exercise runtime
4121// behavior; production spawning is confined to the src/task.rs gateway.
4122#[expect(clippy::disallowed_methods)]
4123mod tests {
4124    use super::*;
4125    use crate::client::{
4126        GetStateOutput, InMemoryExecutionClient, TestResponse, operations_to_checkpoint_log,
4127    };
4128    use crate::engine::{CheckpointLog, CheckpointStatus};
4129    use aws_sdk_lambda::types::Operation;
4130    use std::sync::Arc;
4131
4132    /// A `TypedError` around a `RetryableSerdesError` is still marked, so
4133    /// it ends the invocation, and the fault message holds the wrapped
4134    /// chain without the type name.
4135    #[tokio::test]
4136    async fn typed_retryable_serdes_error_fault_omits_the_type_name() {
4137        let typed = crate::TypedError::with_type(
4138            "Throttled",
4139            crate::serdes::RetryableSerdesError::new("mount timed out"),
4140        );
4141        assert!(crate::serdes::is_retryable_serdes_error(&typed));
4142        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
4143        let ctx = DurableContext::new_root_with_client(
4144            "arn:test".to_owned(),
4145            lambda_runtime::Context::default(),
4146            Arc::new(CheckpointLog::empty()),
4147            Arc::clone(&client) as Arc<dyn ExecutionClient>,
4148            "token0".to_owned(),
4149        );
4150        let fault_ctx = ctx.clone();
4151        let outcome = crate::driver::test_support::outcome_of(
4152            ctx.clone(),
4153            Arc::clone(ctx.task_ownership()),
4154            Arc::clone(ctx.coordinator()),
4155            async move {
4156                let _ = fault_ctx
4157                    .route_serdes_error("serialize", "op-1", Some("s"), Box::new(typed))
4158                    .await;
4159            },
4160        )
4161        .await;
4162        let crate::driver::InvocationOutcome::Fault {
4163            error_type,
4164            message,
4165        } = outcome
4166        else {
4167            unreachable!("expected an invocation fault, got {outcome:?}");
4168        };
4169        assert_eq!(error_type, crate::error::SERDES_FAILED_ERROR_TYPE);
4170        assert_eq!(
4171            message,
4172            "serdes serialize failed for operation \"s\" (op-1): \
4173             retryable serdes failure: mount timed out"
4174        );
4175    }
4176
4177    /// The batch context, its Normal item context and its Flat item
4178    /// context take their levels from the constructors: `new_child` adds 1
4179    /// and `new_flat_child` keeps the level.
4180    #[test]
4181    fn nesting_levels_follow_constructors() {
4182        let (root, _client) = DurableContext::test_context_with_depth(5);
4183        let level = |ctx: &DurableContext| ctx.inner.nesting_level;
4184        assert_eq!(level(&root), 0);
4185        assert_eq!(root.inner.preserve_child_depth, 5);
4186
4187        let batch = root.new_child("1", None);
4188        let normal = batch.new_child("1-1", None);
4189        let flat = batch.new_flat_child("1-2", "parent", None);
4190        assert_eq!((level(&batch), level(&normal), level(&flat)), (1, 2, 1));
4191
4192        let child = root.new_child("2", None);
4193        let batch = child.new_child("2-1", None);
4194        let normal = batch.new_child("2-1-1", None);
4195        let flat = batch.new_flat_child("2-1-2", "parent", None);
4196        assert_eq!((level(&batch), level(&normal), level(&flat)), (2, 3, 2));
4197        for ctx in [&child, &batch, &normal, &flat] {
4198            assert_eq!(
4199                ctx.inner.preserve_child_depth, 5,
4200                "every constructor copies the depth"
4201            );
4202        }
4203
4204        for ctx in [
4205            DurableContext::test_context(),
4206            DurableContext::new_root(
4207                "arn".to_owned(),
4208                lambda_runtime::Context::default(),
4209                Arc::new(CheckpointLog::empty()),
4210            ),
4211        ] {
4212            assert_eq!((level(&ctx), ctx.inner.preserve_child_depth), (0, 0));
4213        }
4214    }
4215
4216    /// `new_virtual_child` keeps the caller's level and depth: 0 under the
4217    /// root and 1 under a child of the root.
4218    #[test]
4219    fn virtual_child_keeps_nesting_level() {
4220        let (root, _client) = DurableContext::test_context_with_depth(5);
4221        let level = |ctx: &DurableContext| ctx.inner.nesting_level;
4222        let under_root = root.new_virtual_child("1", None);
4223        let child = root.new_child("2", None);
4224        let under_child = child.new_virtual_child("2-1", None);
4225        assert_eq!((level(&under_root), level(&under_child)), (0, 1));
4226        for ctx in [&under_root, &under_child] {
4227            assert_eq!(ctx.inner.preserve_child_depth, 5);
4228        }
4229        // The virtual child names its caller's parent and runs in its
4230        // caller's owners.
4231        assert_eq!(under_root.parent_wire_id(), None);
4232        assert_eq!(under_root.owners(), [] as [String; 0]);
4233        assert_eq!(under_child.parent_wire_id(), child.parent_wire_id());
4234        assert_eq!(under_child.owners(), child.owners());
4235    }
4236
4237    /// A context that `new_child_rebuilding` creates with `rebuilding` true
4238    /// is rebuilding, and so is every context derived from it by
4239    /// `new_child`, `new_child_rebuilding`, `new_virtual_child` and
4240    /// `new_flat_child`. A root, a child created with `rebuilding` false,
4241    /// and the contexts derived from it are not.
4242    #[test]
4243    fn rebuilding_state_propagates_to_every_derived_context() {
4244        let root = DurableContext::test_context();
4245        let plain = root.new_child_rebuilding("1", None, false);
4246        let not_rebuilding = [
4247            root.clone(),
4248            root.new_child("2", None),
4249            root.new_virtual_child("3", None),
4250            root.new_flat_child("4", "parent", None),
4251            plain.new_child("1-1", None),
4252            plain.new_virtual_child("1-2", None),
4253            plain.new_flat_child("1-3", "parent", None),
4254            plain.clone(),
4255        ];
4256        for ctx in &not_rebuilding {
4257            assert!(!ctx.is_rebuilding());
4258        }
4259
4260        let rebuilt = root.new_child_rebuilding("5", None, true);
4261        let virtual_child = rebuilt.new_virtual_child("5-1", None);
4262        let rebuilding = [
4263            rebuilt.new_child("5-2", None),
4264            rebuilt.new_child_rebuilding("5-3", None, false),
4265            rebuilt.new_flat_child("5-4", "parent", None),
4266            virtual_child.new_child("5-5", None),
4267            virtual_child.clone(),
4268            rebuilt,
4269        ];
4270        for ctx in &rebuilding {
4271            assert!(ctx.is_rebuilding());
4272        }
4273    }
4274
4275    /// Each row of the issue's table: the context that creates the record,
4276    /// asked with `levels_below` 1, against depths 0, 1, 2 and `u32::MAX`.
4277    #[test]
4278    fn preserves_children_table() {
4279        let depths = [0, 1, 2, u32::MAX];
4280        let row = |make: &dyn Fn(&DurableContext) -> DurableContext| -> Vec<bool> {
4281            depths
4282                .iter()
4283                .map(|&depth| {
4284                    let (root, _client) = DurableContext::test_context_with_depth(depth);
4285                    make(&root).preserves_children(1)
4286                })
4287                .collect()
4288        };
4289        // A top-level child, map or parallel container: level 1.
4290        assert_eq!(row(&|root| root.clone()), [false, true, true, true]);
4291        // An item of a top-level map, Normal nesting: level 2.
4292        assert_eq!(
4293            row(&|root| root.new_child("1", None)),
4294            [false, false, true, true]
4295        );
4296        // A child inside a Normal item: level 3.
4297        assert_eq!(
4298            row(&|root| root.new_child("1", None).new_child("1-1", None)),
4299            [false, false, false, true]
4300        );
4301        // A child inside a Flat item of a top-level map: level 2.
4302        assert_eq!(
4303            row(&|root| root
4304                .new_child("1", None)
4305                .new_flat_child("1-1", "parent", None)),
4306            [false, false, true, true]
4307        );
4308        // A child inside a top-level child: level 2.
4309        assert_eq!(
4310            row(&|root| root.new_child("1", None)),
4311            [false, false, true, true]
4312        );
4313        // Additions saturate instead of overflowing.
4314        let (root, _client) = DurableContext::test_context_with_depth(u32::MAX);
4315        assert!(root.new_child("1", None).preserves_children(u32::MAX));
4316    }
4317
4318    /// Test 19: each combinator method collects the operations its
4319    /// iterator creates, which claim their IDs, and claims no ID itself.
4320    #[test]
4321    fn combinator_methods_collect_without_claiming() {
4322        let ctx = DurableContext::new_root(
4323            "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
4324            lambda_runtime::Context::default(),
4325            Arc::new(CheckpointLog::empty()),
4326        );
4327        // A lazy iterator: each step claims its ID when the combinator
4328        // method collects it.
4329        let inputs = || {
4330            let ctx = &ctx;
4331            (0..2).map(move |n| ctx.step(move |_| async move { Ok(n) }).future())
4332        };
4333        let try_join_all = ctx.try_join_all(inputs());
4334        let join_all = ctx.join_all(inputs());
4335        let select_ok = ctx.select_ok(inputs());
4336        let race = ctx.race(inputs());
4337        assert_eq!(
4338            ctx.mint_id().positional(),
4339            "9",
4340            "the eight steps claimed 1 to 8, the combinators nothing"
4341        );
4342        drop((try_join_all, join_all, select_ok, race));
4343    }
4344
4345    /// A claim outside any `run_body` scope checks only ownership, and the
4346    /// owner thread passes it.
4347    #[test]
4348    fn claim_without_active_body_records_nothing() {
4349        let ctx = DurableContext::test_context();
4350        let id = ctx.claim_operation_id("step");
4351        assert_eq!(id.positional(), "1");
4352        assert!(ctx.coordinator().fatal_error().is_none());
4353    }
4354
4355    /// A claim from a thread that does not own the context, with no body
4356    /// active, records the ownership error and still claims the next ID.
4357    /// The owner's claim before it records nothing.
4358    #[test]
4359    fn claim_from_foreign_thread_records_fatal() {
4360        let ctx = DurableContext::test_context();
4361        assert_eq!(ctx.claim_operation_id("step").positional(), "1");
4362        assert!(ctx.coordinator().fatal_error().is_none());
4363
4364        let id = std::thread::scope(|scope| {
4365            scope
4366                .spawn(|| ctx.claim_operation_id("step"))
4367                .join()
4368                .expect("the claim does not panic")
4369        });
4370        assert_eq!(id.positional(), "2");
4371        let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4372        assert_eq!(fatal.error_type, "Error");
4373        assert_eq!(fatal.error_message, foreign_caller_message("step"));
4374    }
4375
4376    /// `enforce_task_ownership` passes for the owner thread and records
4377    /// nothing. From a thread that does not own the context it returns an
4378    /// `ExecutionFailed` step error and records the ownership error.
4379    #[test]
4380    fn enforce_task_ownership_refuses_foreign_thread() {
4381        let ctx = DurableContext::test_context();
4382        assert!(ctx.enforce_task_ownership().is_ok());
4383        assert!(ctx.coordinator().fatal_error().is_none());
4384
4385        let result = std::thread::scope(|scope| {
4386            scope
4387                .spawn(|| ctx.enforce_task_ownership())
4388                .join()
4389                .expect("the check does not panic")
4390        });
4391        let err = result.expect_err("a foreign thread is refused");
4392        assert!(matches!(
4393            err.kind(),
4394            OperationErrorKind::Step(step) if matches!(step.kind(), StepErrorKind::ExecutionFailed)
4395        ));
4396        // The `StepError` is the operation error's source, and the ownership
4397        // message is the step error's source.
4398        let source = std::error::Error::source(&err)
4399            .and_then(std::error::Error::source)
4400            .map(ToString::to_string);
4401        assert!(
4402            source
4403                .as_deref()
4404                .is_some_and(|msg| msg.contains("which does not own the context")),
4405            "unexpected source: {source:?}"
4406        );
4407        let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4408        assert_eq!(fatal.error_type, "Error");
4409        assert_eq!(fatal.error_message, foreign_caller_message("execute"));
4410    }
4411
4412    /// A claim on the context whose body is active passes.
4413    #[tokio::test]
4414    async fn claim_in_own_body_records_nothing() {
4415        let ctx = DurableContext::test_context();
4416        let id = run_body(ActiveBody::Context(String::new()), async {
4417            ctx.claim_operation_id("step")
4418        })
4419        .await;
4420        assert_eq!(id.positional(), "1");
4421        assert!(ctx.coordinator().fatal_error().is_none());
4422    }
4423
4424    /// A claim on the context whose body is active, made by a raw task that
4425    /// does not own the context, still runs the ownership check: the
4426    /// matching body does not authorize the caller. It records the
4427    /// ownership error and still claims the next ID.
4428    #[tokio::test]
4429    async fn claim_in_own_body_on_raw_task_records_fatal() {
4430        let ctx = DurableContext::test_context();
4431        assert_eq!(ctx.claim_operation_id("step").positional(), "1");
4432        let foreign = ctx.clone();
4433        let id = tokio::spawn(run_body(ActiveBody::Context(String::new()), async move {
4434            foreign.claim_operation_id("step")
4435        }))
4436        .await
4437        .expect("the claim does not panic");
4438        assert_eq!(id.positional(), "2");
4439        let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4440        assert_eq!(fatal.error_type, "Error");
4441        assert_eq!(fatal.error_message, foreign_caller_message("step"));
4442    }
4443
4444    /// A claim on another context than the active body records message A
4445    /// and still claims the ID.
4446    #[tokio::test]
4447    async fn claim_in_other_body_records_fatal() {
4448        let ctx = DurableContext::test_context();
4449        let id = run_body(ActiveBody::Context("3".to_owned()), async {
4450            ctx.claim_operation_id("wait")
4451        })
4452        .await;
4453        assert_eq!(id.positional(), "1");
4454        let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4455        assert_eq!(fatal.error_type, "Error");
4456        assert_eq!(
4457            fatal.error_message,
4458            "context usage error in \"wait\": the operation was called on context \"root\", \
4459             but the SDK is running the body of context \"3\". Inside run_in_child_context, \
4460             with_retry, map and parallel, call operations on the context passed to the closure."
4461        );
4462    }
4463
4464    /// A claim inside a step body records message B.
4465    #[tokio::test]
4466    async fn claim_in_step_body_records_fatal() {
4467        let ctx = DurableContext::test_context();
4468        let _id = run_body(ActiveBody::Step("4".to_owned()), async {
4469            ctx.claim_operation_id("map")
4470        })
4471        .await;
4472        let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4473        assert_eq!(fatal.error_type, "Error");
4474        assert_eq!(
4475            fatal.error_message,
4476            "context usage error in \"map\": the operation was called inside the body of \
4477             operation \"4\". A step body or a wait_for_condition check cannot call durable \
4478             operations; use run_in_child_context to group operations."
4479        );
4480    }
4481
4482    /// Helper: builds a Step operation.
4483    fn make_step_op(id: &str, result: &str) -> Operation {
4484        Operation::builder()
4485            .id(id)
4486            .r#type(aws_sdk_lambda::types::OperationType::Step)
4487            .status(aws_sdk_lambda::types::OperationStatus::Succeeded)
4488            .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
4489            .step_details(
4490                aws_sdk_lambda::types::StepDetails::builder()
4491                    .result(result)
4492                    .build(),
4493            )
4494            .build()
4495            .unwrap()
4496    }
4497
4498    /// Tests that `checkpoint_updates` paginates when the checkpoint response
4499    /// has a `next_marker`: calling `get_state` to fetch all remaining
4500    /// operations and merging them into the checkpoint log.
4501    #[tokio::test]
4502    async fn checkpoint_updates_paginates_on_marker() {
4503        // The full state has 3 operations (what get_state returns).
4504        let all_ops = vec![
4505            make_step_op("step-1", "\"r1\""),
4506            make_step_op("step-2", "\"r2\""),
4507            make_step_op("step-3", "\"r3\""),
4508        ];
4509        let client = Arc::new(InMemoryExecutionClient::new(all_ops));
4510
4511        // Enqueue a paginated checkpoint response: returns only step-1
4512        // but signals more pages via next_marker.
4513        let page1_ops = vec![make_step_op("step-1", "\"r1\"")];
4514        client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
4515            page1_ops,
4516            "page-2-token".to_owned(),
4517        ));
4518
4519        let log = Arc::new(CheckpointLog::empty());
4520        let ctx = DurableContext::new_root_with_client(
4521            "arn:test".to_owned(),
4522            lambda_runtime::Context::default(),
4523            Arc::clone(&log),
4524            client.clone(),
4525            "initial-token".to_owned(),
4526        );
4527
4528        // Perform a checkpoint with no updates (we just want to exercise pagination).
4529        let result = ctx.checkpoint_updates(Vec::new()).await;
4530        assert!(result.is_ok());
4531
4532        // The checkpoint log should now contain ALL operations from get_state.
4533        assert!(log.get("step-1").is_some(), "step-1 must be in the log");
4534        assert!(
4535            log.get("step-2").is_some(),
4536            "step-2 must be in the log (paginated)"
4537        );
4538        assert!(
4539            log.get("step-3").is_some(),
4540            "step-3 must be in the log (paginated)"
4541        );
4542
4543        // get_state should have been called once for pagination.
4544        let get_state_count = *client.get_state_call_count.lock().unwrap();
4545        assert_eq!(
4546            get_state_count, 1,
4547            "get_state must be called for pagination"
4548        );
4549    }
4550
4551    /// Mock client for the concurrent pagination/token race test: every
4552    /// `checkpoint` validates and rotates the token, always returns a
4553    /// pagination marker, and `get_state` records whether the token it was
4554    /// handed is still the client's CURRENT token.
4555    #[derive(Debug)]
4556    struct PaginatedTokenValidatingClient {
4557        current_token: std::sync::Mutex<String>,
4558        counter: std::sync::atomic::AtomicU32,
4559        stale_checkpoint_tokens: std::sync::atomic::AtomicU32,
4560        stale_get_state_tokens: std::sync::atomic::AtomicU32,
4561        get_state_calls: std::sync::atomic::AtomicU32,
4562    }
4563
4564    impl PaginatedTokenValidatingClient {
4565        fn new(initial_token: &str) -> Self {
4566            Self {
4567                current_token: std::sync::Mutex::new(initial_token.to_owned()),
4568                counter: std::sync::atomic::AtomicU32::new(0),
4569                stale_checkpoint_tokens: std::sync::atomic::AtomicU32::new(0),
4570                stale_get_state_tokens: std::sync::atomic::AtomicU32::new(0),
4571                get_state_calls: std::sync::atomic::AtomicU32::new(0),
4572            }
4573        }
4574    }
4575
4576    impl ExecutionClient for PaginatedTokenValidatingClient {
4577        fn checkpoint(
4578            &self,
4579            _execution_arn: &str,
4580            checkpoint_token: &str,
4581            _updates: Vec<OperationUpdate>,
4582        ) -> std::pin::Pin<
4583            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
4584        > {
4585            let submitted = checkpoint_token.to_owned();
4586            Box::pin(async move {
4587                // Widen the race window like the real network call would.
4588                tokio::task::yield_now().await;
4589                let mut current = self
4590                    .current_token
4591                    .lock()
4592                    .unwrap_or_else(std::sync::PoisonError::into_inner);
4593                if *current != submitted {
4594                    self.stale_checkpoint_tokens
4595                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4596                    return Err(ExecutionClientError::non_retryable(format!(
4597                        "stale checkpoint token: expected {current}, got {submitted}"
4598                    )));
4599                }
4600                let n = self
4601                    .counter
4602                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4603                let next = format!("token-{}", n + 1);
4604                current.clone_from(&next);
4605                drop(current);
4606                Ok(CheckpointOutput {
4607                    checkpoint_token: Some(next),
4608                    updated_operations: Vec::new(),
4609                    // Every response paginated: forces the marker-triggered
4610                    // get_state on every checkpoint.
4611                    next_marker: Some("more-pages".to_owned()),
4612                })
4613            })
4614        }
4615
4616        fn get_state(
4617            &self,
4618            _execution_arn: &str,
4619            checkpoint_token: &str,
4620            _marker: Option<&str>,
4621        ) -> std::pin::Pin<
4622            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
4623        > {
4624            let submitted = checkpoint_token.to_owned();
4625            Box::pin(async move {
4626                self.get_state_calls
4627                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4628                // Yield so a concurrent branch would have every chance to
4629                // checkpoint (and rotate the token) if the caller released
4630                // the token lock too early.
4631                tokio::task::yield_now().await;
4632                tokio::task::yield_now().await;
4633                let current = self
4634                    .current_token
4635                    .lock()
4636                    .unwrap_or_else(std::sync::PoisonError::into_inner);
4637                if *current != submitted {
4638                    self.stale_get_state_tokens
4639                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4640                }
4641                drop(current);
4642                Ok(GetStateOutput::new(Vec::new(), None))
4643            })
4644        }
4645    }
4646
4647    /// Regression test (issue #5): concurrent checkpoints whose responses
4648    /// carry a pagination marker must perform the marker-triggered
4649    /// `get_state` while STILL holding the checkpoint-token lock. If the
4650    /// lock is dropped first, a concurrent branch checkpoints, consumes and
4651    /// rotates the token, and the paginated `get_state` runs with a stale
4652    /// token. The mock's `get_state` validates the token it receives
4653    /// against the client's current token.
4654    #[tokio::test]
4655    async fn concurrent_paginated_checkpoints_get_state_uses_current_token() {
4656        let client = Arc::new(PaginatedTokenValidatingClient::new("token-0"));
4657        let log = Arc::new(CheckpointLog::empty());
4658        let ctx = DurableContext::new_root_with_client(
4659            "arn:test".to_owned(),
4660            lambda_runtime::Context::default(),
4661            log,
4662            Arc::clone(&client) as Arc<dyn ExecutionClient>,
4663            "token-0".to_owned(),
4664        );
4665
4666        // Race several concurrent writes, each of which paginates. Every
4667        // one must succeed (no stale checkpoint token) and every get_state
4668        // must observe the then-current token. The batch writer sends one
4669        // request at a time, so the race calls `write_unchecked`, which
4670        // takes the token lock directly.
4671        let (r1, r2, r3, r4) = tokio::join!(
4672            ctx.write_unchecked(Vec::new(), &[], None),
4673            ctx.write_unchecked(Vec::new(), &[], None),
4674            ctx.write_unchecked(Vec::new(), &[], None),
4675            ctx.write_unchecked(Vec::new(), &[], None),
4676        );
4677        assert!(r1.is_ok(), "checkpoint 1 failed: {r1:?}");
4678        assert!(r2.is_ok(), "checkpoint 2 failed: {r2:?}");
4679        assert!(r3.is_ok(), "checkpoint 3 failed: {r3:?}");
4680        assert!(r4.is_ok(), "checkpoint 4 failed: {r4:?}");
4681
4682        assert_eq!(
4683            client
4684                .get_state_calls
4685                .load(std::sync::atomic::Ordering::SeqCst),
4686            4,
4687            "every paginated checkpoint must trigger a get_state"
4688        );
4689        assert_eq!(
4690            client
4691                .stale_checkpoint_tokens
4692                .load(std::sync::atomic::Ordering::SeqCst),
4693            0,
4694            "no checkpoint may run with a stale token"
4695        );
4696        assert_eq!(
4697            client
4698                .stale_get_state_tokens
4699                .load(std::sync::atomic::Ordering::SeqCst),
4700            0,
4701            "the marker-triggered get_state must run with the current token \
4702             (token lock held through the paginated fetch)"
4703        );
4704    }
4705
4706    /// The write after the drive goes through the batch writer: 251 updates
4707    /// go out in two requests, in their order.
4708    #[tokio::test]
4709    async fn checkpoint_updates_after_drive_uses_batch_limits() {
4710        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
4711        let ctx = DurableContext::new_root_with_client(
4712            "arn:test".to_owned(),
4713            lambda_runtime::Context::default(),
4714            Arc::new(CheckpointLog::empty()),
4715            Arc::clone(&client) as Arc<dyn ExecutionClient>,
4716            "token-0".to_owned(),
4717        );
4718        let ids: Vec<String> = (0..251).map(|n| format!("op-{n}")).collect();
4719        let updates = ids.iter().map(|id| make_update(id)).collect();
4720        ctx.checkpoint_updates_after_drive(updates)
4721            .await
4722            .expect("the write succeeds");
4723        assert_eq!(*client.checkpoint_call_count.lock().unwrap(), 2);
4724        let sent: Vec<String> = client
4725            .recorded_updates()
4726            .iter()
4727            .map(|update| update.id().to_owned())
4728            .collect();
4729        assert_eq!(sent, ids);
4730    }
4731
4732    /// Tests that `checkpoint_updates` does NOT call `get_state` when there
4733    /// is no pagination marker.
4734    #[tokio::test]
4735    async fn checkpoint_updates_no_pagination_when_no_marker() {
4736        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
4737        // Default response has no marker.
4738
4739        let log = Arc::new(CheckpointLog::empty());
4740        let ctx = DurableContext::new_root_with_client(
4741            "arn:test".to_owned(),
4742            lambda_runtime::Context::default(),
4743            log,
4744            client.clone(),
4745            "initial-token".to_owned(),
4746        );
4747
4748        let result = ctx.checkpoint_updates(Vec::new()).await;
4749        assert!(result.is_ok());
4750
4751        let get_state_count = *client.get_state_call_count.lock().unwrap();
4752        assert_eq!(
4753            get_state_count, 0,
4754            "get_state must NOT be called without a marker"
4755        );
4756    }
4757
4758    /// Tests that the bootstrap pagination path works: when the initial
4759    /// state has a `NextMarker`, `get_state` is called to fetch the full
4760    /// operation set, and all operations appear in the checkpoint log.
4761    #[tokio::test]
4762    async fn bootstrap_pagination_fetches_all_pages() {
4763        // Simulate a paginated initial state: page 1 has step-1,
4764        // and get_state returns the full set (step-1 + step-2 + step-3).
4765        let all_ops = vec![
4766            make_step_op("step-1", "\"result-1\""),
4767            make_step_op("step-2", "\"result-2\""),
4768            make_step_op("step-3", "\"result-3\""),
4769        ];
4770        let client: Arc<dyn ExecutionClient> = Arc::new(InMemoryExecutionClient::new(all_ops));
4771
4772        // The full log comes from get_state when the initial state is paginated.
4773        let full_state = client.get_state("arn:test", "token", None).await;
4774        assert!(full_state.is_ok());
4775        let full_state = full_state.unwrap();
4776
4777        let log = operations_to_checkpoint_log(&full_state.operations);
4778        assert!(log.get("step-1").is_some());
4779        assert!(log.get("step-2").is_some());
4780        assert!(log.get("step-3").is_some());
4781
4782        // Verify that the results are correct.
4783        let r2 = log.get("step-2").unwrap();
4784        assert_eq!(r2.result.as_deref(), Some("\"result-2\""));
4785        assert_eq!(r2.status, CheckpointStatus::Succeeded);
4786    }
4787
4788    // Non-determinism detection tests
4789
4790    #[test]
4791    fn validate_replay_identity_passes_when_types_match() {
4792        let log = Arc::new(CheckpointLog::empty());
4793        let ctx = DurableContext::new_root(
4794            "arn:test".to_owned(),
4795            lambda_runtime::Context::default(),
4796            log,
4797        );
4798        let record = CheckpointRecord {
4799            id: "wire-1".to_owned(),
4800            status: CheckpointStatus::Succeeded,
4801            result: Some("42".to_owned()),
4802            error_type: None,
4803            error_message: None,
4804            error_data: None,
4805            stack_trace: None,
4806            attempt: 0,
4807            invoke_result: None,
4808            invoke_error_type: None,
4809            invoke_error_message: None,
4810            invoke_error_data: None,
4811            invoke_stack_trace: None,
4812            replay_children: false,
4813            callback_id: None,
4814            op_type: Some("Step".to_owned()),
4815            sub_type: Some("Step".to_owned()),
4816            due_at: None,
4817            op_name: Some("my-step".to_owned()),
4818            parent_id: None,
4819            start_timestamp: None,
4820            end_timestamp: None,
4821        };
4822
4823        let result =
4824            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("my-step"));
4825        assert!(result.is_ok());
4826    }
4827
4828    // Targeted checkpoint accessors
4829
4830    /// Builds a context whose checkpoint log holds one record at positional
4831    /// ID `"1"` (keyed by its wire ID, matching the production log shape).
4832    fn ctx_with_record_at_1(record: CheckpointRecord) -> DurableContext {
4833        let wire_id = crate::engine::compute_wire_id_public("1");
4834        let log = Arc::new(CheckpointLog::from_records(vec![(wire_id, record)]));
4835        DurableContext::new_root(
4836            "arn:test".to_owned(),
4837            lambda_runtime::Context::default(),
4838            log,
4839        )
4840    }
4841
4842    /// A record at a virtual child's ID fails with a recorded mismatch whose
4843    /// claimed identity is `virtual Context`; a record without a type reads
4844    /// as `Unknown`; no record passes.
4845    #[test]
4846    fn reject_record_at_virtual_child_messages() {
4847        let record = |op_type: Option<&str>| CheckpointRecord {
4848            id: "wire-1".to_owned(),
4849            status: CheckpointStatus::Succeeded,
4850            result: None,
4851            error_type: None,
4852            error_message: None,
4853            error_data: None,
4854            stack_trace: None,
4855            attempt: 0,
4856            invoke_result: None,
4857            invoke_error_type: None,
4858            invoke_error_message: None,
4859            invoke_error_data: None,
4860            invoke_stack_trace: None,
4861            replay_children: false,
4862            callback_id: None,
4863            op_type: op_type.map(str::to_owned),
4864            sub_type: Some("RunInChildContext".to_owned()),
4865            due_at: None,
4866            op_name: Some("slot".to_owned()),
4867            parent_id: None,
4868            start_timestamp: None,
4869            end_timestamp: None,
4870        };
4871        for (op_type, expected) in [
4872            (
4873                Some("Context"),
4874                "expected Context/RunInChildContext named \"slot\"",
4875            ),
4876            (None, "expected Unknown/RunInChildContext named \"slot\""),
4877        ] {
4878            let ctx = ctx_with_record_at_1(record(op_type));
4879            let op_id = ctx.mint_id();
4880            let err = ctx
4881                .reject_record_at_virtual_child(&op_id, "RunInChildContext", Some("slot"))
4882                .expect_err("a record at the slot");
4883            let message = crate::error::chain_string(&err);
4884            assert!(message.contains(expected), "{message}");
4885            assert!(
4886                message.contains("got virtual Context/RunInChildContext named \"slot\""),
4887                "{message}"
4888            );
4889            let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
4890            assert_eq!(fatal.error_type, "NonDeterministicExecutionError");
4891            let next = ctx.mint_id();
4892            assert!(
4893                ctx.reject_record_at_virtual_child(&next, "RunInChildContext", None)
4894                    .is_ok()
4895            );
4896        }
4897    }
4898
4899    #[test]
4900    fn checkpoint_view_validated_returns_view_on_identity_match() {
4901        let ctx = ctx_with_record_at_1(CheckpointRecord {
4902            id: "wire-1".to_owned(),
4903            status: CheckpointStatus::Succeeded,
4904            result: Some("42".to_owned()),
4905            error_type: None,
4906            error_message: None,
4907            error_data: None,
4908            stack_trace: None,
4909            attempt: 2,
4910            invoke_result: None,
4911            invoke_error_type: None,
4912            invoke_error_message: None,
4913            invoke_error_data: None,
4914            invoke_stack_trace: None,
4915            replay_children: true,
4916            callback_id: None,
4917            op_type: Some("Step".to_owned()),
4918            sub_type: Some("Step".to_owned()),
4919            due_at: None,
4920            op_name: Some("my-step".to_owned()),
4921            parent_id: None,
4922            start_timestamp: None,
4923            end_timestamp: None,
4924        });
4925
4926        let result =
4927            ctx.checkpoint_view_validated("1", "wire-1", "Step", Some("Step"), Some("my-step"));
4928        assert!(result.is_ok());
4929        let view = result.unwrap();
4930        assert!(view.is_some());
4931        let view = view.unwrap();
4932        assert_eq!(view.status, CheckpointStatus::Succeeded);
4933        assert_eq!(view.attempt, 2);
4934        assert!(view.replay_children);
4935
4936        // No record at another position: Ok(None), nothing to validate.
4937        let live = ctx.checkpoint_view_validated("2", "wire-2", "Step", Some("Step"), None);
4938        assert!(matches!(live, Ok(None)));
4939    }
4940
4941    #[test]
4942    fn checkpoint_view_validated_errors_on_identity_mismatch() {
4943        let ctx = ctx_with_record_at_1(CheckpointRecord {
4944            id: "wire-1".to_owned(),
4945            status: CheckpointStatus::Succeeded,
4946            result: None,
4947            error_type: None,
4948            error_message: None,
4949            error_data: None,
4950            stack_trace: None,
4951            attempt: 0,
4952            invoke_result: None,
4953            invoke_error_type: None,
4954            invoke_error_message: None,
4955            invoke_error_data: None,
4956            invoke_stack_trace: None,
4957            replay_children: false,
4958            callback_id: None,
4959            op_type: Some("Step".to_owned()),
4960            sub_type: Some("Step".to_owned()),
4961            due_at: None,
4962            op_name: None,
4963            parent_id: None,
4964            start_timestamp: None,
4965            end_timestamp: None,
4966        });
4967
4968        // Claimed as Wait but checkpointed as Step → the same
4969        // NonDeterministicExecution error `validate_replay_identity` builds.
4970        let result = ctx.checkpoint_view_validated("1", "wire-1", "Wait", Some("Wait"), None);
4971        assert!(result.is_err());
4972        let err = result.unwrap_err();
4973        assert!(matches!(
4974            err.kind(),
4975            OperationErrorKind::NonDeterministicExecution(_)
4976        ));
4977        let display = format!("{err:#}");
4978        assert!(
4979            display.contains("Step") && display.contains("Wait"),
4980            "expected both identities in error: {display}"
4981        );
4982    }
4983
4984    /// Replay reaching a record whose status this SDK version does not
4985    /// recognize must fail the execution naming the raw status (issue
4986    /// #45), never return the view for the operation to act on.
4987    #[test]
4988    #[expect(clippy::panic)] // reason: test assertions over non_exhaustive kinds
4989    fn checkpoint_view_validated_fails_execution_on_unrecognized_status() {
4990        let ctx = ctx_with_record_at_1(CheckpointRecord {
4991            id: "wire-1".to_owned(),
4992            status: CheckpointStatus::Unknown("PAUSED".to_owned()),
4993            result: None,
4994            error_type: None,
4995            error_message: None,
4996            error_data: None,
4997            stack_trace: None,
4998            attempt: 0,
4999            invoke_result: None,
5000            invoke_error_type: None,
5001            invoke_error_message: None,
5002            invoke_error_data: None,
5003            invoke_stack_trace: None,
5004            replay_children: false,
5005            callback_id: None,
5006            op_type: Some("Step".to_owned()),
5007            sub_type: Some("Step".to_owned()),
5008            due_at: None,
5009            op_name: None,
5010            parent_id: None,
5011            start_timestamp: None,
5012            end_timestamp: None,
5013        });
5014
5015        assert!(
5016            ctx.coordinator().fatal_error().is_none(),
5017            "no fatal recorded before the replay check"
5018        );
5019
5020        // Identity matches: the failure is the status, not the identity.
5021        let result = ctx.checkpoint_view_validated("1", "wire-1", "Step", Some("Step"), None);
5022        assert!(result.is_err());
5023        let err = result.unwrap_err();
5024        let OperationErrorKind::NonDeterministicExecution(nde) = err.kind() else {
5025            panic!("expected NonDeterministicExecution, got {:?}", err.kind());
5026        };
5027        let NonDeterministicExecutionErrorKind::UnrecognizedStatus(details) = nde.kind() else {
5028            panic!("expected UnrecognizedStatus, got {:?}", nde.kind());
5029        };
5030        assert_eq!(details.wire_id(), "wire-1");
5031        assert_eq!(details.status(), "PAUSED");
5032        let display = format!("{err:#}");
5033        assert!(
5034            display.contains("PAUSED"),
5035            "error must name the raw status: {display}"
5036        );
5037
5038        // The fatal slot is written so the invocation driver fails the
5039        // execution even if the returned `Err` is swallowed downstream.
5040        let fatal = ctx
5041            .coordinator()
5042            .fatal_error()
5043            .expect("unrecognized status must record a fatal error");
5044        assert_eq!(fatal.error_type, "NonDeterministicExecutionError");
5045        assert!(
5046            fatal.error_message.contains("PAUSED"),
5047            "fatal message must carry the raw status: {}",
5048            fatal.error_message
5049        );
5050    }
5051
5052    /// A record with a recognized status and matching identity records no
5053    /// fatal and returns the view: the #45 guard fires only on `Unknown`.
5054    #[test]
5055    fn checkpoint_view_validated_known_status_records_no_fatal() {
5056        let ctx = ctx_with_record_at_1(CheckpointRecord {
5057            id: "wire-1".to_owned(),
5058            status: CheckpointStatus::TimedOut,
5059            result: None,
5060            error_type: None,
5061            error_message: None,
5062            error_data: None,
5063            stack_trace: None,
5064            attempt: 0,
5065            invoke_result: None,
5066            invoke_error_type: None,
5067            invoke_error_message: None,
5068            invoke_error_data: None,
5069            invoke_stack_trace: None,
5070            replay_children: false,
5071            callback_id: None,
5072            op_type: Some("Step".to_owned()),
5073            sub_type: Some("Step".to_owned()),
5074            due_at: None,
5075            op_name: None,
5076            parent_id: None,
5077            start_timestamp: None,
5078            end_timestamp: None,
5079        });
5080
5081        let result = ctx.checkpoint_view_validated("1", "wire-1", "Step", Some("Step"), None);
5082        assert!(matches!(result, Ok(Some(_))));
5083        assert!(ctx.coordinator().fatal_error().is_none());
5084    }
5085
5086    #[test]
5087    fn targeted_getters_return_single_fields() {
5088        let ctx = ctx_with_record_at_1(CheckpointRecord {
5089            id: "wire-1".to_owned(),
5090            status: CheckpointStatus::Failed,
5091            result: Some(r#""state""#.to_owned()),
5092            error_type: Some("SomeError".to_owned()),
5093            error_message: Some("it broke".to_owned()),
5094            error_data: None,
5095            stack_trace: None,
5096            attempt: 5,
5097            invoke_result: None,
5098            invoke_error_type: None,
5099            invoke_error_message: None,
5100            invoke_error_data: None,
5101            invoke_stack_trace: None,
5102            replay_children: false,
5103            callback_id: Some("cb-42".to_owned()),
5104            op_type: None,
5105            sub_type: None,
5106            due_at: None,
5107            op_name: None,
5108            parent_id: None,
5109            start_timestamp: None,
5110            end_timestamp: None,
5111        });
5112
5113        assert!(ctx.has_checkpoint_record("1"));
5114        assert!(!ctx.has_checkpoint_record("2"));
5115
5116        assert_eq!(
5117            ctx.checkpoint_result_payload("1").as_deref(),
5118            Some(r#""state""#)
5119        );
5120        assert!(ctx.checkpoint_result_payload("2").is_none());
5121
5122        assert_eq!(ctx.checkpoint_callback_id("1").as_deref(), Some("cb-42"));
5123
5124        let wire = ctx.checkpoint_wire_error("1");
5125        let wire = wire.expect("record 1 exists");
5126        assert_eq!(wire.error_type(), Some("SomeError"));
5127        assert_eq!(wire.error_message(), Some("it broke"));
5128        assert!(ctx.checkpoint_wire_error("2").is_none());
5129
5130        let view = ctx.checkpoint_status_view("1");
5131        assert!(view.is_some());
5132        let view = view.unwrap();
5133        assert_eq!(view.status, CheckpointStatus::Failed);
5134        assert_eq!(view.attempt, 5);
5135    }
5136
5137    #[test]
5138    fn checkpoint_terminal_replay_projects_payload_and_error_fields() {
5139        let ctx = ctx_with_record_at_1(CheckpointRecord {
5140            id: "wire-1".to_owned(),
5141            status: CheckpointStatus::Failed,
5142            result: Some(r#""summary""#.to_owned()),
5143            error_type: Some("BatchError".to_owned()),
5144            error_message: Some("batch broke".to_owned()),
5145            error_data: Some(r#"{"code":7}"#.to_owned()),
5146            stack_trace: Some(vec!["frame-a".to_owned()]),
5147            attempt: 3,
5148            invoke_result: Some("never-projected".to_owned()),
5149            invoke_error_type: Some("never-projected".to_owned()),
5150            invoke_error_message: Some("never-projected".to_owned()),
5151            invoke_error_data: None,
5152            invoke_stack_trace: None,
5153            replay_children: true,
5154            callback_id: Some("cb-99".to_owned()),
5155            op_type: Some("Context".to_owned()),
5156            sub_type: Some("Map".to_owned()),
5157            due_at: None,
5158            op_name: Some("batch".to_owned()),
5159            parent_id: None,
5160            start_timestamp: None,
5161            end_timestamp: None,
5162        });
5163
5164        let snapshot = ctx.checkpoint_terminal_replay("1");
5165        assert_eq!(
5166            snapshot,
5167            Some(TerminalReplaySnapshot {
5168                status: CheckpointStatus::Failed,
5169                replay_children: true,
5170                result: Some(r#""summary""#.to_owned()),
5171                error_message: Some("batch broke".to_owned()),
5172                error_type: Some("BatchError".to_owned()),
5173                error_data: Some(r#"{"code":7}"#.to_owned()),
5174                stack_trace: Some(vec!["frame-a".to_owned()]),
5175            })
5176        );
5177
5178        // No record → no snapshot.
5179        assert!(ctx.checkpoint_terminal_replay("2").is_none());
5180    }
5181
5182    #[test]
5183    fn validate_replay_identity_fails_on_type_mismatch() {
5184        let log = Arc::new(CheckpointLog::empty());
5185        let ctx = DurableContext::new_root(
5186            "arn:test".to_owned(),
5187            lambda_runtime::Context::default(),
5188            log,
5189        );
5190        let record = CheckpointRecord {
5191            id: "wire-1".to_owned(),
5192            status: CheckpointStatus::Succeeded,
5193            result: None,
5194            error_type: None,
5195            error_message: None,
5196            error_data: None,
5197            stack_trace: None,
5198            attempt: 0,
5199            invoke_result: None,
5200            invoke_error_type: None,
5201            invoke_error_message: None,
5202            invoke_error_data: None,
5203            invoke_stack_trace: None,
5204            replay_children: false,
5205            callback_id: None,
5206            op_type: Some("Step".to_owned()),
5207            sub_type: Some("Step".to_owned()),
5208            due_at: None,
5209            op_name: None,
5210            parent_id: None,
5211            start_timestamp: None,
5212            end_timestamp: None,
5213        };
5214
5215        // Claimed as Wait but checkpointed as Step → error
5216        let result = ctx.validate_replay_identity(&record, "wire-1", "Wait", Some("Wait"), None);
5217        assert!(result.is_err());
5218        let err = result.unwrap_err();
5219        assert!(matches!(
5220            err.kind(),
5221            OperationErrorKind::NonDeterministicExecution(_)
5222        ));
5223        let display = format!("{err:#}");
5224        assert!(
5225            display.contains("Step"),
5226            "expected Step in error: {display}"
5227        );
5228        assert!(
5229            display.contains("Wait"),
5230            "expected Wait in error: {display}"
5231        );
5232    }
5233
5234    #[test]
5235    fn validate_replay_identity_fails_on_subtype_mismatch() {
5236        let log = Arc::new(CheckpointLog::empty());
5237        let ctx = DurableContext::new_root(
5238            "arn:test".to_owned(),
5239            lambda_runtime::Context::default(),
5240            log,
5241        );
5242        let record = CheckpointRecord {
5243            id: "wire-1".to_owned(),
5244            status: CheckpointStatus::Succeeded,
5245            result: None,
5246            error_type: None,
5247            error_message: None,
5248            error_data: None,
5249            stack_trace: None,
5250            attempt: 0,
5251            invoke_result: None,
5252            invoke_error_type: None,
5253            invoke_error_message: None,
5254            invoke_error_data: None,
5255            invoke_stack_trace: None,
5256            replay_children: false,
5257            callback_id: None,
5258            op_type: Some("Step".to_owned()),
5259            sub_type: Some("Step".to_owned()),
5260            due_at: None,
5261            op_name: None,
5262            parent_id: None,
5263            start_timestamp: None,
5264            end_timestamp: None,
5265        };
5266
5267        // Same op_type but different sub_type → error
5268        let result =
5269            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("WaitForCondition"), None);
5270        assert!(result.is_err());
5271        let err = result.unwrap_err();
5272        assert!(matches!(
5273            err.kind(),
5274            OperationErrorKind::NonDeterministicExecution(_)
5275        ));
5276    }
5277
5278    #[test]
5279    fn validate_replay_identity_fails_on_name_mismatch() {
5280        let log = Arc::new(CheckpointLog::empty());
5281        let ctx = DurableContext::new_root(
5282            "arn:test".to_owned(),
5283            lambda_runtime::Context::default(),
5284            log,
5285        );
5286        let record = CheckpointRecord {
5287            id: "wire-1".to_owned(),
5288            status: CheckpointStatus::Succeeded,
5289            result: None,
5290            error_type: None,
5291            error_message: None,
5292            error_data: None,
5293            stack_trace: None,
5294            attempt: 0,
5295            invoke_result: None,
5296            invoke_error_type: None,
5297            invoke_error_message: None,
5298            invoke_error_data: None,
5299            invoke_stack_trace: None,
5300            replay_children: false,
5301            callback_id: None,
5302            op_type: Some("Step".to_owned()),
5303            sub_type: Some("Step".to_owned()),
5304            due_at: None,
5305            op_name: Some("fetch-user".to_owned()),
5306            parent_id: None,
5307            start_timestamp: None,
5308            end_timestamp: None,
5309        };
5310
5311        // Same type but different name → error
5312        let result = ctx.validate_replay_identity(
5313            &record,
5314            "wire-1",
5315            "Step",
5316            Some("Step"),
5317            Some("fetch-order"),
5318        );
5319        assert!(result.is_err());
5320        let err = result.unwrap_err();
5321        let display = format!("{err:#}");
5322        assert!(
5323            display.contains("fetch-user"),
5324            "expected name in error: {display}"
5325        );
5326        assert!(
5327            display.contains("fetch-order"),
5328            "expected claimed name in error: {display}"
5329        );
5330    }
5331
5332    #[test]
5333    fn validate_replay_identity_skips_when_no_identity_stored() {
5334        let log = Arc::new(CheckpointLog::empty());
5335        let ctx = DurableContext::new_root(
5336            "arn:test".to_owned(),
5337            lambda_runtime::Context::default(),
5338            log,
5339        );
5340        let record = CheckpointRecord {
5341            id: "wire-1".to_owned(),
5342            status: CheckpointStatus::Succeeded,
5343            result: Some("1".to_owned()),
5344            error_type: None,
5345            error_message: None,
5346            error_data: None,
5347            stack_trace: None,
5348            attempt: 0,
5349            invoke_result: None,
5350            invoke_error_type: None,
5351            invoke_error_message: None,
5352            invoke_error_data: None,
5353            invoke_stack_trace: None,
5354            replay_children: false,
5355            callback_id: None,
5356            op_type: None,
5357            sub_type: None,
5358            due_at: None,
5359            op_name: None,
5360            parent_id: None,
5361            start_timestamp: None,
5362            end_timestamp: None,
5363        };
5364
5365        // Even though claimed type differs, validation passes because the
5366        // record has no identity to compare against.
5367        let result = ctx.validate_replay_identity(&record, "wire-1", "Wait", Some("Wait"), None);
5368        assert!(result.is_ok());
5369    }
5370
5371    #[test]
5372    fn validate_replay_identity_case_insensitive_type_match() {
5373        let log = Arc::new(CheckpointLog::empty());
5374        let ctx = DurableContext::new_root(
5375            "arn:test".to_owned(),
5376            lambda_runtime::Context::default(),
5377            log,
5378        );
5379        let record = CheckpointRecord {
5380            id: "wire-1".to_owned(),
5381            status: CheckpointStatus::Succeeded,
5382            result: None,
5383            error_type: None,
5384            error_message: None,
5385            error_data: None,
5386            stack_trace: None,
5387            attempt: 0,
5388            invoke_result: None,
5389            invoke_error_type: None,
5390            invoke_error_message: None,
5391            invoke_error_data: None,
5392            invoke_stack_trace: None,
5393            replay_children: false,
5394            callback_id: None,
5395            op_type: Some("STEP".to_owned()),
5396            sub_type: Some("STEP".to_owned()),
5397            due_at: None,
5398            op_name: None,
5399            parent_id: None,
5400            start_timestamp: None,
5401            end_timestamp: None,
5402        };
5403
5404        // Wire format uses UPPER_CASE: validation is case-insensitive.
5405        let result = ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), None);
5406        assert!(result.is_ok());
5407    }
5408
5409    /// Regression test (issue #6): the inline JSON envelope path stores the
5410    /// raw wire value `CHAINED_INVOKE` (with underscore) while the SDK
5411    /// claims `ChainedInvoke` (`PascalCase`). A case-insensitive comparison
5412    /// alone rejects every non-paginated replayed invoke as
5413    /// non-deterministic; canonicalization must accept it.
5414    #[test]
5415    fn validate_replay_identity_inline_chained_invoke_replay_matches() {
5416        let log = Arc::new(CheckpointLog::empty());
5417        let ctx = DurableContext::new_root(
5418            "arn:test".to_owned(),
5419            lambda_runtime::Context::default(),
5420            log,
5421        );
5422        let record = CheckpointRecord {
5423            id: "wire-1".to_owned(),
5424            status: CheckpointStatus::Succeeded,
5425            result: None,
5426            error_type: None,
5427            error_message: None,
5428            error_data: None,
5429            stack_trace: None,
5430            attempt: 0,
5431            invoke_result: Some("\"ok\"".to_owned()),
5432            invoke_error_type: None,
5433            invoke_error_message: None,
5434            invoke_error_data: None,
5435            invoke_stack_trace: None,
5436            replay_children: false,
5437            callback_id: None,
5438            // Exactly what parse_single_operation stores from the inline
5439            // envelope: the raw wire `Type` value.
5440            op_type: Some("CHAINED_INVOKE".to_owned()),
5441            sub_type: Some("ChainedInvoke".to_owned()),
5442            due_at: None,
5443            op_name: None,
5444            parent_id: None,
5445            start_timestamp: None,
5446            end_timestamp: None,
5447        };
5448
5449        let result = ctx.validate_replay_identity(
5450            &record,
5451            "wire-1",
5452            "ChainedInvoke",
5453            Some("ChainedInvoke"),
5454            None,
5455        );
5456        assert!(
5457            result.is_ok(),
5458            "inline CHAINED_INVOKE record must match claimed ChainedInvoke: {result:?}"
5459        );
5460    }
5461
5462    /// Canonicalization must not weaken detection: a genuinely different
5463    /// type still mismatches after canonicalization.
5464    #[test]
5465    fn validate_replay_identity_canonicalized_types_still_mismatch() {
5466        let log = Arc::new(CheckpointLog::empty());
5467        let ctx = DurableContext::new_root(
5468            "arn:test".to_owned(),
5469            lambda_runtime::Context::default(),
5470            log,
5471        );
5472        let record = CheckpointRecord {
5473            id: "wire-1".to_owned(),
5474            status: CheckpointStatus::Succeeded,
5475            result: None,
5476            error_type: None,
5477            error_message: None,
5478            error_data: None,
5479            stack_trace: None,
5480            attempt: 0,
5481            invoke_result: None,
5482            invoke_error_type: None,
5483            invoke_error_message: None,
5484            invoke_error_data: None,
5485            invoke_stack_trace: None,
5486            replay_children: false,
5487            callback_id: None,
5488            op_type: Some("CHAINED_INVOKE".to_owned()),
5489            sub_type: Some("ChainedInvoke".to_owned()),
5490            due_at: None,
5491            op_name: None,
5492            parent_id: None,
5493            start_timestamp: None,
5494            end_timestamp: None,
5495        };
5496
5497        let result =
5498            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("ChainedInvoke"), None);
5499        assert!(
5500            result.is_err(),
5501            "Step claimed against CHAINED_INVOKE record must mismatch"
5502        );
5503    }
5504
5505    /// Both spellings of every known operation type canonicalize to the
5506    /// same wire constant, and unknown types stay distinguishable.
5507    #[test]
5508    fn canonical_op_type_bridges_wire_and_pascal_spellings() {
5509        for (pascal, wire) in [
5510            ("Callback", "CALLBACK"),
5511            ("ChainedInvoke", "CHAINED_INVOKE"),
5512            ("Context", "CONTEXT"),
5513            ("Execution", "EXECUTION"),
5514            ("Step", "STEP"),
5515            ("Wait", "WAIT"),
5516        ] {
5517            assert_eq!(
5518                canonical_op_type(pascal),
5519                canonical_op_type(wire),
5520                "{pascal} and {wire} must canonicalize identically"
5521            );
5522        }
5523        assert_ne!(
5524            canonical_op_type("Step"),
5525            canonical_op_type("ChainedInvoke"),
5526            "distinct types must stay distinct"
5527        );
5528    }
5529
5530    /// Builds a checkpoint record carrying full identity fields for the
5531    /// Some↔None conformance tests below.
5532    fn record_with_identity(
5533        op_type: Option<&str>,
5534        sub_type: Option<&str>,
5535        op_name: Option<&str>,
5536    ) -> CheckpointRecord {
5537        CheckpointRecord {
5538            id: "wire-1".to_owned(),
5539            status: CheckpointStatus::Succeeded,
5540            result: None,
5541            error_type: None,
5542            error_message: None,
5543            error_data: None,
5544            stack_trace: None,
5545            attempt: 0,
5546            invoke_result: None,
5547            invoke_error_type: None,
5548            invoke_error_message: None,
5549            invoke_error_data: None,
5550            invoke_stack_trace: None,
5551            replay_children: false,
5552            callback_id: None,
5553            op_type: op_type.map(ToOwned::to_owned),
5554            sub_type: sub_type.map(ToOwned::to_owned),
5555            due_at: None,
5556            op_name: op_name.map(ToOwned::to_owned),
5557            parent_id: None,
5558            start_timestamp: None,
5559            end_timestamp: None,
5560        }
5561    }
5562
5563    fn test_ctx() -> DurableContext {
5564        DurableContext::new_root(
5565            "arn:test".to_owned(),
5566            lambda_runtime::Context::default(),
5567            Arc::new(CheckpointLog::empty()),
5568        )
5569    }
5570
5571    #[test]
5572    fn validate_replay_identity_fails_when_stored_name_removed() {
5573        // Record was checkpointed with a name; the claim carries none.
5574        // Removing `.name(...)` changes replay identity and must be flagged,
5575        // otherwise a reordered unnamed same-type operation could consume
5576        // this checkpoint silently.
5577        let ctx = test_ctx();
5578        let record = record_with_identity(Some("Step"), Some("Step"), Some("my-step"));
5579
5580        let result = ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), None);
5581        let err = result.unwrap_err();
5582        assert!(matches!(
5583            err.kind(),
5584            OperationErrorKind::NonDeterministicExecution(_)
5585        ));
5586        let display = format!("{err:#}");
5587        assert!(
5588            display.contains("my-step"),
5589            "expected stored name in error: {display}"
5590        );
5591    }
5592
5593    #[test]
5594    fn validate_replay_identity_fails_when_name_added() {
5595        // Record was checkpointed without a name (but with identity); the
5596        // claim now carries one. The None↔Some direction is a mismatch too.
5597        let ctx = test_ctx();
5598        let record = record_with_identity(Some("Step"), Some("Step"), None);
5599
5600        let result =
5601            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("new-name"));
5602        let err = result.unwrap_err();
5603        assert!(matches!(
5604            err.kind(),
5605            OperationErrorKind::NonDeterministicExecution(_)
5606        ));
5607        let display = format!("{err:#}");
5608        assert!(
5609            display.contains("new-name"),
5610            "expected claimed name in error: {display}"
5611        );
5612    }
5613
5614    #[test]
5615    fn validate_replay_identity_fails_when_subtype_dropped() {
5616        // Sub-type is compared as a complete Option in both directions.
5617        let ctx = test_ctx();
5618        let record = record_with_identity(Some("Context"), Some("Map"), None);
5619
5620        let result = ctx.validate_replay_identity(&record, "wire-1", "Context", None, None);
5621        assert!(
5622            result.is_err(),
5623            "expected mismatch when stored sub-type is dropped from the claim"
5624        );
5625
5626        // And the reverse direction: record without a sub-type, claim with one.
5627        let record = record_with_identity(Some("Context"), None, None);
5628        let result = ctx.validate_replay_identity(&record, "wire-1", "Context", Some("Map"), None);
5629        assert!(
5630            result.is_err(),
5631            "expected mismatch when a sub-type is added to the claim"
5632        );
5633    }
5634
5635    #[test]
5636    fn validate_replay_identity_untyped_record_skips_name_and_subtype() {
5637        // A record genuinely lacking identity (no op_type) is the ONLY
5638        // lenient path: nothing is compared, whatever the claim carries.
5639        let ctx = test_ctx();
5640        let record = record_with_identity(None, None, None);
5641
5642        let result =
5643            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("any-name"));
5644        assert!(result.is_ok());
5645    }
5646
5647    #[test]
5648    fn validate_replay_identity_empty_claimed_name_matches_stored_none() {
5649        // An empty claimed name and a record with no `Name` are the same
5650        // identity. An unchanged handler must NOT be rejected on resume.
5651        let ctx = test_ctx();
5652        let record = record_with_identity(Some("Context"), Some("MapIteration"), None);
5653
5654        let result = ctx.validate_replay_identity(
5655            &record,
5656            "wire-1",
5657            "Context",
5658            Some("MapIteration"),
5659            Some(""),
5660        );
5661        assert!(
5662            result.is_ok(),
5663            "claimed empty name must match a stored None: {result:?}"
5664        );
5665    }
5666
5667    #[test]
5668    fn validate_replay_identity_empty_stored_name_matches_claimed_none() {
5669        // The reverse direction: a backend that stored an empty name string
5670        // must match a claim carrying no name.
5671        let ctx = test_ctx();
5672        let record = record_with_identity(Some("Step"), Some("Step"), Some(""));
5673
5674        let result = ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), None);
5675        assert!(
5676            result.is_ok(),
5677            "stored empty name must match a claimed None: {result:?}"
5678        );
5679    }
5680
5681    #[test]
5682    fn validate_replay_identity_empty_name_still_mismatches_real_name() {
5683        // Normalization must not weaken detection: an empty claim against a
5684        // real stored name (and the reverse) is still a mismatch.
5685        let ctx = test_ctx();
5686        let record = record_with_identity(Some("Step"), Some("Step"), Some("real"));
5687        let result =
5688            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some(""));
5689        assert!(
5690            result.is_err(),
5691            "empty claim vs stored 'real' must mismatch"
5692        );
5693
5694        let record = record_with_identity(Some("Step"), Some("Step"), None);
5695        let result =
5696            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("real"));
5697        assert!(
5698            result.is_err(),
5699            "claimed 'real' vs stored None must mismatch"
5700        );
5701    }
5702
5703    #[test]
5704    fn validate_replay_identity_mismatch_records_fatal_error() {
5705        // A mismatch must record the execution-fatal error on the shared
5706        // suspension-signal slot so the invocation driver fails the execution
5707        // even when the returned `Err` is swallowed downstream.
5708        let ctx = test_ctx();
5709        let record = record_with_identity(Some("Step"), Some("Step"), Some("alpha"));
5710
5711        assert!(
5712            ctx.coordinator().fatal_error().is_none(),
5713            "no fatal recorded before validation"
5714        );
5715        let result =
5716            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("beta"));
5717        assert!(result.is_err());
5718
5719        let fatal = ctx
5720            .coordinator()
5721            .fatal_error()
5722            .expect("mismatch must record a fatal error");
5723        assert_eq!(fatal.error_type, "NonDeterministicExecutionError");
5724        assert!(
5725            fatal.error_message.contains("alpha") && fatal.error_message.contains("beta"),
5726            "fatal message must name both identities: {}",
5727            fatal.error_message
5728        );
5729    }
5730
5731    #[test]
5732    fn validate_replay_identity_pass_records_no_fatal() {
5733        let ctx = test_ctx();
5734        let record = record_with_identity(Some("Step"), Some("Step"), Some("same"));
5735
5736        let result =
5737            ctx.validate_replay_identity(&record, "wire-1", "Step", Some("Step"), Some("same"));
5738        assert!(result.is_ok());
5739        assert!(
5740            ctx.coordinator().fatal_error().is_none(),
5741            "a passing validation must not record a fatal error"
5742        );
5743    }
5744
5745    // Debug redaction tests
5746
5747    #[test]
5748    fn debug_output_redacts_checkpoint_token() {
5749        let secret_token = "super-secret-credential-value-12345";
5750        let log = Arc::new(CheckpointLog::empty());
5751        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5752        let ctx = DurableContext::new_root_with_client(
5753            "arn:aws:lambda:us-east-1:123456789012:function:my-fn".to_owned(),
5754            lambda_runtime::Context::default(),
5755            log,
5756            client,
5757            secret_token.to_owned(),
5758        );
5759
5760        let debug_output = format!("{ctx:?}");
5761
5762        // The actual token value MUST NOT appear in debug output.
5763        assert!(
5764            !debug_output.contains(secret_token),
5765            "checkpoint_token value leaked in Debug output: {debug_output}"
5766        );
5767
5768        // The redacted placeholder MUST appear.
5769        assert!(
5770            debug_output.contains("<redacted>"),
5771            "expected '<redacted>' in Debug output: {debug_output}"
5772        );
5773
5774        // Useful fields MUST be present.
5775        assert!(
5776            debug_output.contains("arn:aws:lambda:us-east-1:123456789012:function:my-fn"),
5777            "expected execution_arn in Debug output: {debug_output}"
5778        );
5779    }
5780
5781    /// Every context derived from the root shares the root's plugins: the
5782    /// child of `new_child` and the flat child of `new_flat_child`. The test
5783    /// compares the `Arc`s by address.
5784    #[test]
5785    fn derived_contexts_share_the_root_plugins() {
5786        let plugins = crate::plugin::test_support::invocation_plugins();
5787        let root = DurableContext::new_root_with_client_and_defaults(
5788            "arn:test".to_owned(),
5789            lambda_runtime::Context::default(),
5790            Arc::new(CheckpointLog::empty()),
5791            Arc::new(InMemoryExecutionClient::new(Vec::new())),
5792            "token-0".to_owned(),
5793            Duration::ZERO,
5794            0,
5795            Some(Arc::clone(&plugins)),
5796        );
5797        let same = |ctx: &DurableContext| {
5798            ctx.plugins()
5799                .is_some_and(|shared| std::ptr::eq(shared, Arc::as_ptr(&plugins)))
5800        };
5801        assert!(same(&root), "the root holds the plugins");
5802        let child = root.new_child("1", None);
5803        assert!(same(&child), "new_child copies the plugins");
5804        assert!(
5805            same(&child.new_child("1-1", None)),
5806            "a grandchild shares them"
5807        );
5808        assert!(
5809            same(&root.new_flat_child("2", "parent", None)),
5810            "new_flat_child copies the plugins"
5811        );
5812        assert!(
5813            DurableContext::test_context().plugins().is_none(),
5814            "a root constructor without the parameter sets none"
5815        );
5816    }
5817
5818    // Checkpoint coalescing tests
5819
5820    /// Builds a root context with a checkpoint-coalescing delay, backed by
5821    /// the in-memory client.
5822    fn coalescing_ctx(client: Arc<dyn ExecutionClient>, delay: Duration) -> DurableContext {
5823        DurableContext::new_root_with_client_and_defaults(
5824            "arn:test".to_owned(),
5825            lambda_runtime::Context::default(),
5826            Arc::new(CheckpointLog::empty()),
5827            client,
5828            "token-0".to_owned(),
5829            delay,
5830            0,
5831            None,
5832        )
5833    }
5834
5835    /// Helper: builds a bare step START update with the given wire ID.
5836    fn make_update(id: &str) -> OperationUpdate {
5837        OperationUpdate::builder()
5838            .id(id.to_owned())
5839            .r#type(aws_sdk_lambda::types::OperationType::Step)
5840            .action(OperationAction::Start)
5841            .build()
5842            .expect("all required OperationUpdate fields set")
5843    }
5844
5845    /// Without a Tokio runtime the batch flusher cannot start: every
5846    /// contributor receives an invocation-scoped error instead of waiting
5847    /// forever, and nothing is written.
5848    #[test]
5849    fn batch_flush_without_runtime_returns_invocation_error() {
5850        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5851        let ctx = coalescing_ctx(client.clone(), Duration::ZERO);
5852        let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
5853            ctx.checkpoint_updates(vec![make_update("op-a")]).await
5854        });
5855        let std::task::Poll::Ready(Err(err)) = polled else {
5856            unreachable!("expected an immediate error, got {polled:?}");
5857        };
5858        assert_eq!(err.scope(), ExecutionClientErrorScope::Invocation);
5859        assert!(
5860            err.to_string().contains(crate::task::NO_RUNTIME_MESSAGE),
5861            "unexpected error: {err}"
5862        );
5863        assert!(client.recorded_updates().is_empty(), "nothing is written");
5864    }
5865
5866    /// A flusher that cannot be spawned publishes nothing for a batch that
5867    /// another flusher already claimed: that flusher publishes the
5868    /// batch's result.
5869    #[test]
5870    fn unspawned_flush_of_a_claimed_batch_publishes_nothing() {
5871        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5872        let ctx = coalescing_ctx(client.clone(), Duration::from_secs(60));
5873        let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
5874            let coalescer = Arc::clone(&ctx.inner.coalescer);
5875            let (batch, _) = coalescer
5876                .join(
5877                    ctx.admission(),
5878                    ctx.coordinator(),
5879                    ctx.owners(),
5880                    vec![TrackedUpdate::capture(make_update("op-a"), 1)],
5881                )
5882                .expect("admitted");
5883            let claimed = coalescer.take_batch(&batch);
5884            ctx.spawn_batch_flush(&coalescer, &batch, Duration::ZERO);
5885            (claimed.is_some(), batch.result_clone().is_none())
5886        });
5887        assert!(
5888            matches!(polled, std::task::Poll::Ready((true, true))),
5889            "{polled:?}"
5890        );
5891        assert!(client.recorded_updates().is_empty(), "nothing is written");
5892    }
5893
5894    /// Two concurrent checkpoint calls inside the delay window coalesce
5895    /// into ONE client call carrying both updates.
5896    #[tokio::test(start_paused = true)]
5897    async fn coalesced_checkpoints_batch_into_one_call() {
5898        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5899        let ctx = coalescing_ctx(client.clone(), Duration::from_millis(50));
5900
5901        let (a, b) = tokio::join!(
5902            ctx.checkpoint_updates(vec![make_update("op-a")]),
5903            ctx.checkpoint_updates(vec![make_update("op-b")]),
5904        );
5905        assert!(a.is_ok(), "first coalesced caller succeeds");
5906        assert!(b.is_ok(), "second coalesced caller succeeds");
5907
5908        let call_count = *client.checkpoint_call_count.lock().unwrap();
5909        assert_eq!(call_count, 1, "both updates must share one API call");
5910
5911        let ids: Vec<String> = client
5912            .recorded_updates()
5913            .iter()
5914            .map(|u| u.id.clone())
5915            .collect();
5916        assert_eq!(
5917            ids,
5918            vec!["op-a".to_owned(), "op-b".to_owned()],
5919            "the single call carries both updates in join order"
5920        );
5921    }
5922
5923    /// An urgent checkpoint (the callback-creation flush point) does not
5924    /// wait out the delay window: it flushes immediately, carrying any
5925    /// previously buffered updates with it.
5926    #[tokio::test(start_paused = true)]
5927    async fn urgent_checkpoint_flushes_without_waiting_the_window() {
5928        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5929        // A window long enough that waiting it out would be visible in
5930        // paused time.
5931        let ctx = coalescing_ctx(client.clone(), Duration::from_hours(1));
5932
5933        let start = tokio::time::Instant::now();
5934        ctx.submit_checkpoint_updates_urgent(vec![make_update("op-urgent")])
5935            .await
5936            .expect("urgent checkpoint succeeds");
5937
5938        assert_eq!(
5939            start.elapsed(),
5940            Duration::ZERO,
5941            "urgent flush must not wait for the coalescing window"
5942        );
5943        let call_count = *client.checkpoint_call_count.lock().unwrap();
5944        assert_eq!(call_count, 1);
5945    }
5946
5947    /// `flush_pending_checkpoints` (the suspension / end-of-invocation
5948    /// flush point) drains a buffered batch long before its window
5949    /// elapses, and the buffered caller observes the flushed result.
5950    #[tokio::test(start_paused = true)]
5951    async fn flush_pending_checkpoints_drains_the_buffer() {
5952        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5953        let ctx = coalescing_ctx(client.clone(), Duration::from_hours(1));
5954
5955        let start = tokio::time::Instant::now();
5956        let buffered = tokio::spawn({
5957            let ctx = ctx.clone();
5958            async move {
5959                ctx.checkpoint_updates(vec![make_update("op-buffered")])
5960                    .await
5961            }
5962        });
5963        // Let the spawned caller join the batch (well under the window).
5964        tokio::time::sleep(Duration::from_millis(1)).await;
5965
5966        ctx.flush_pending_checkpoints().await;
5967
5968        let result = buffered.await.expect("buffered caller task completes");
5969        assert!(result.is_ok(), "buffered caller observes the flush result");
5970        assert!(
5971            start.elapsed() < Duration::from_secs(1),
5972            "flush must not wait for the one-hour window"
5973        );
5974        let call_count = *client.checkpoint_call_count.lock().unwrap();
5975        assert_eq!(call_count, 1);
5976    }
5977
5978    /// Without a configured delay, `checkpoint_updates` writes immediately:
5979    /// one call per checkpoint when each write is awaited before the next.
5980    #[tokio::test(start_paused = true)]
5981    async fn no_delay_checkpoints_write_immediately() {
5982        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
5983        let ctx = DurableContext::new_root_with_client(
5984            "arn:test".to_owned(),
5985            lambda_runtime::Context::default(),
5986            Arc::new(CheckpointLog::empty()),
5987            client.clone(),
5988            "token-0".to_owned(),
5989        );
5990
5991        let start = tokio::time::Instant::now();
5992        ctx.checkpoint_updates(vec![make_update("op-1")])
5993            .await
5994            .expect("first write succeeds");
5995        ctx.checkpoint_updates(vec![make_update("op-2")])
5996            .await
5997            .expect("second write succeeds");
5998
5999        assert_eq!(start.elapsed(), Duration::ZERO);
6000        let call_count = *client.checkpoint_call_count.lock().unwrap();
6001        assert_eq!(call_count, 2, "no coalescing without a configured delay");
6002    }
6003
6004    /// A test client whose `checkpoint` calls block until released,
6005    /// delegating to an [`InMemoryExecutionClient`] once the gate opens.
6006    /// Lets tests hold a batched write "in flight" deterministically.
6007    #[derive(Debug)]
6008    struct GatedClient {
6009        inner: InMemoryExecutionClient,
6010        gate: tokio::sync::Semaphore,
6011        /// Snapshot of update-ID lists, one entry per checkpoint call, in
6012        /// call order.
6013        calls: std::sync::Mutex<Vec<Vec<String>>>,
6014    }
6015
6016    impl GatedClient {
6017        fn new() -> Self {
6018            Self {
6019                inner: InMemoryExecutionClient::new(Vec::new()),
6020                gate: tokio::sync::Semaphore::new(0),
6021                calls: std::sync::Mutex::new(Vec::new()),
6022            }
6023        }
6024
6025        /// Allows one blocked (or future) checkpoint call to proceed.
6026        fn release_one(&self) {
6027            self.gate.add_permits(1);
6028        }
6029
6030        fn call_ids(&self) -> Vec<Vec<String>> {
6031            self.calls
6032                .lock()
6033                .unwrap_or_else(std::sync::PoisonError::into_inner)
6034                .clone()
6035        }
6036    }
6037
6038    impl ExecutionClient for GatedClient {
6039        fn checkpoint(
6040            &self,
6041            execution_arn: &str,
6042            checkpoint_token: &str,
6043            updates: Vec<OperationUpdate>,
6044        ) -> std::pin::Pin<
6045            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6046        > {
6047            let arn = execution_arn.to_owned();
6048            let token = checkpoint_token.to_owned();
6049            Box::pin(async move {
6050                // Block until the test releases the gate. The permit is
6051                // consumed (forgotten) so each call needs its own release.
6052                let permit = self.gate.acquire().await.map_err(|e| {
6053                    ExecutionClientError::non_retryable(format!("gate closed: {e}"))
6054                })?;
6055                permit.forget();
6056                self.calls
6057                    .lock()
6058                    .unwrap_or_else(std::sync::PoisonError::into_inner)
6059                    .push(updates.iter().map(|u| u.id.clone()).collect());
6060                self.inner.checkpoint(&arn, &token, updates).await
6061            })
6062        }
6063
6064        fn get_state(
6065            &self,
6066            execution_arn: &str,
6067            checkpoint_token: &str,
6068            marker: Option<&str>,
6069        ) -> std::pin::Pin<
6070            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6071        > {
6072            self.inner
6073                .get_state(execution_arn, checkpoint_token, marker)
6074        }
6075    }
6076
6077    /// REGRESSION (flush contract): once a delay timer claims a batch and
6078    /// its write is in flight, `flush_pending_checkpoints` must NOT return
6079    /// until that write finishes: the wrapper reports
6080    /// PENDING/SUCCEEDED/FAILED right after the flush, and an unawaited
6081    /// in-flight checkpoint would cross that boundary.
6082    #[tokio::test(start_paused = true)]
6083    async fn flush_waits_for_in_flight_timer_claimed_batch() {
6084        let client = Arc::new(GatedClient::new());
6085        let ctx = coalescing_ctx(
6086            Arc::clone(&client) as Arc<dyn ExecutionClient>,
6087            Duration::from_millis(10),
6088        );
6089
6090        // A buffered contributor: its delay timer fires at +10ms, claiming
6091        // the batch and starting a write that blocks on the gate.
6092        let contributor = tokio::spawn({
6093            let ctx = ctx.clone();
6094            async move {
6095                ctx.checkpoint_updates(vec![make_update("op-in-flight")])
6096                    .await
6097            }
6098        });
6099        // Advance past the window so the timer claims the batch and the
6100        // write task blocks inside the gated client.
6101        tokio::time::sleep(Duration::from_millis(20)).await;
6102
6103        // The end-of-invocation flush must now WAIT for that in-flight
6104        // write. Prove it stays pending while the gate is closed: in
6105        // paused time, the timeout can only fire because every task is
6106        // blocked on the (non-timer) gate.
6107        let mut flush = tokio::spawn({
6108            let ctx = ctx.clone();
6109            async move { ctx.flush_pending_checkpoints().await }
6110        });
6111        let raced = tokio::time::timeout(Duration::from_mins(1), &mut flush).await;
6112        assert!(
6113            raced.is_err(),
6114            "flush returned while the timer-claimed batch write was still in flight — \
6115             the flush barrier is broken"
6116        );
6117
6118        // Release the write; now both the contributor and the flush finish.
6119        client.release_one();
6120        flush.await.expect("flush task completes");
6121        contributor
6122            .await
6123            .expect("contributor task completes")
6124            .expect("contributor observes the published write");
6125        assert_eq!(
6126            client.call_ids(),
6127            vec![vec!["op-in-flight".to_owned()]],
6128            "exactly the claimed batch was written, once"
6129        );
6130    }
6131
6132    /// A test client that FAILS every checkpoint call retryably, holding
6133    /// the FIRST call at a gate so the test can order a second batch's
6134    /// queuing against the in-flight failing write. Calls after the first
6135    /// fail immediately (no gate), so a regression that performs an extra
6136    /// write shows up as an extra recorded call instead of a hang.
6137    #[derive(Debug)]
6138    struct GatedRetryableFailClient {
6139        gate: tokio::sync::Semaphore,
6140        first: std::sync::atomic::AtomicBool,
6141        /// Snapshot of update-ID lists, one entry per checkpoint call, in
6142        /// call order.
6143        calls: std::sync::Mutex<Vec<Vec<String>>>,
6144    }
6145
6146    impl GatedRetryableFailClient {
6147        fn new() -> Self {
6148            Self {
6149                gate: tokio::sync::Semaphore::new(0),
6150                first: std::sync::atomic::AtomicBool::new(true),
6151                calls: std::sync::Mutex::new(Vec::new()),
6152            }
6153        }
6154
6155        /// Releases the gated first call so its rejection lands.
6156        fn release_first(&self) {
6157            self.gate.add_permits(1);
6158        }
6159
6160        fn call_ids(&self) -> Vec<Vec<String>> {
6161            self.calls
6162                .lock()
6163                .unwrap_or_else(std::sync::PoisonError::into_inner)
6164                .clone()
6165        }
6166
6167        fn call_count(&self) -> usize {
6168            self.calls
6169                .lock()
6170                .unwrap_or_else(std::sync::PoisonError::into_inner)
6171                .len()
6172        }
6173    }
6174
6175    impl ExecutionClient for GatedRetryableFailClient {
6176        fn checkpoint(
6177            &self,
6178            _execution_arn: &str,
6179            _checkpoint_token: &str,
6180            updates: Vec<OperationUpdate>,
6181        ) -> std::pin::Pin<
6182            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6183        > {
6184            Box::pin(async move {
6185                self.calls
6186                    .lock()
6187                    .unwrap_or_else(std::sync::PoisonError::into_inner)
6188                    .push(updates.iter().map(|u| u.id.clone()).collect());
6189                if self.first.swap(false, std::sync::atomic::Ordering::SeqCst) {
6190                    // Hold the first call in flight until the test
6191                    // releases it; the permit is consumed so a second
6192                    // call never blocks here.
6193                    if let Ok(permit) = self.gate.acquire().await {
6194                        permit.forget();
6195                    }
6196                }
6197                Err(ExecutionClientError::from_retryable(
6198                    "injected retryable checkpoint failure".to_owned(),
6199                ))
6200            })
6201        }
6202
6203        fn get_state(
6204            &self,
6205            _execution_arn: &str,
6206            _checkpoint_token: &str,
6207            _marker: Option<&str>,
6208        ) -> std::pin::Pin<
6209            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6210        > {
6211            Box::pin(async {
6212                Err(ExecutionClientError::new_non_retryable(
6213                    "get_state not used in this test",
6214                ))
6215            })
6216        }
6217    }
6218
6219    /// A flusher QUEUED behind a failing batch write performs no
6220    /// checkpoint call. Batch A's write is in flight (held at the client's
6221    /// gate) when batch B opens and queues its flusher on the writer lock;
6222    /// A then fails retryably. The failure sets the latch and drops B's
6223    /// batch, and B's flusher, which claimed nothing or reaches the
6224    /// checkpoint-token lock after the latch, sends nothing and publishes
6225    /// nothing. No contributor resolves, and the first failure is the
6226    /// recorded one.
6227    #[tokio::test(start_paused = true)]
6228    async fn queued_batch_flush_after_retryable_failure_writes_nothing() {
6229        let client = Arc::new(GatedRetryableFailClient::new());
6230        let ctx = coalescing_ctx(
6231            Arc::clone(&client) as Arc<dyn ExecutionClient>,
6232            Duration::ZERO,
6233        );
6234
6235        // Batch A: its zero-delay flusher claims the batch under the
6236        // writer lock and starts the write, which is held at the gate.
6237        let contributor_a = tokio::spawn({
6238            let ctx = ctx.clone();
6239            async move { ctx.checkpoint_updates(vec![make_update("op-a")]).await }
6240        });
6241        while client.call_count() < 1 {
6242            tokio::time::sleep(Duration::from_millis(1)).await;
6243        }
6244
6245        // Batch B opens while A's write is in flight; its flusher queues
6246        // on the writer lock behind A.
6247        let contributor_b = tokio::spawn({
6248            let ctx = ctx.clone();
6249            async move { ctx.checkpoint_updates(vec![make_update("op-b")]).await }
6250        });
6251        // Let B join the buffer and its flusher reach the writer lock.
6252        tokio::time::sleep(Duration::from_millis(5)).await;
6253
6254        // A's held write now fails retryably; B's queued flusher runs
6255        // right after, under the same writer lock.
6256        client.release_first();
6257
6258        let both = async { tokio::join!(contributor_a, contributor_b) };
6259        assert!(
6260            tokio::time::timeout(Duration::from_secs(60), both)
6261                .await
6262                .is_err(),
6263            "no contributor resolves"
6264        );
6265        assert_eq!(
6266            client.call_ids(),
6267            vec![vec!["op-a".to_owned()]],
6268            "batch B causes no checkpoint call after batch A's failure"
6269        );
6270        assert!(ctx.queued_update_ids().is_empty(), "B's batch is dropped");
6271        let first = ctx.coordinator().write_failure().expect("latched");
6272        assert_eq!(
6273            first.error_type,
6274            crate::error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE
6275        );
6276        assert_eq!(
6277            first.error_message,
6278            "[Unrecoverable Invocation] Checkpoint failed: injected retryable checkpoint failure"
6279        );
6280    }
6281
6282    /// A retryable checkpoint failure records the fault JS gives it: the
6283    /// type `CheckpointUnrecoverableInvocationError` and the message
6284    /// `[Unrecoverable Invocation] Checkpoint failed: {m}`, and fails the
6285    /// invocation.
6286    #[test]
6287    fn checkpoint_failure_fatal_of_a_retryable_error() {
6288        let err = ExecutionClientError::new(ExecutionClientErrorScope::Invocation, "throttled");
6289        let fatal = checkpoint_failure_fatal(&err);
6290        assert_eq!(fatal.error_type, "CheckpointUnrecoverableInvocationError");
6291        assert_eq!(
6292            fatal.error_message,
6293            "[Unrecoverable Invocation] Checkpoint failed: throttled"
6294        );
6295        assert!(matches!(fatal.severity, FatalSeverity::FailInvocation));
6296    }
6297
6298    /// A fan-out exceeding one request's operation-count limit is split
6299    /// into multiple ordered requests rather than one oversized call.
6300    #[tokio::test(start_paused = true)]
6301    async fn coalesced_fanout_splits_into_size_capped_requests() {
6302        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6303        let limits = BatchLimits {
6304            max_operations: 2,
6305            max_payload_bytes: usize::MAX,
6306        };
6307        let ctx = DurableContext::new_root_with_client_and_coalescer(
6308            "arn:test".to_owned(),
6309            lambda_runtime::Context::default(),
6310            Arc::new(CheckpointLog::empty()),
6311            client.clone(),
6312            "token-0".to_owned(),
6313            CheckpointCoalescer::with_limits(Duration::from_millis(50), limits),
6314        );
6315
6316        // Five contributors join one coalescing window.
6317        let results = tokio::join!(
6318            ctx.checkpoint_updates(vec![make_update("op-1")]),
6319            ctx.checkpoint_updates(vec![make_update("op-2")]),
6320            ctx.checkpoint_updates(vec![make_update("op-3")]),
6321            ctx.checkpoint_updates(vec![make_update("op-4")]),
6322            ctx.checkpoint_updates(vec![make_update("op-5")]),
6323        );
6324        for outcome in [results.0, results.1, results.2, results.3, results.4] {
6325            assert!(outcome.is_ok(), "every contributor observes success");
6326        }
6327
6328        let call_count = *client.checkpoint_call_count.lock().unwrap();
6329        assert_eq!(
6330            call_count, 3,
6331            "five updates under a two-op cap must go out as 2+2+1 requests"
6332        );
6333        let ids: Vec<String> = client
6334            .recorded_updates()
6335            .iter()
6336            .map(|u| u.id.clone())
6337            .collect();
6338        assert_eq!(
6339            ids,
6340            vec!["op-1", "op-2", "op-3", "op-4", "op-5"],
6341            "join order is preserved across the split requests"
6342        );
6343    }
6344
6345    /// The default zero window: checkpoints arriving while an
6346    /// earlier write is in flight batch into ONE follow-up call, and no
6347    /// artificial delay is added anywhere.
6348    #[tokio::test(start_paused = true)]
6349    async fn batching_mode_batches_writes_behind_in_flight_call() {
6350        let client = Arc::new(GatedClient::new());
6351        let ctx = coalescing_ctx(
6352            Arc::clone(&client) as Arc<dyn ExecutionClient>,
6353            Duration::ZERO,
6354        );
6355
6356        // First checkpoint drives immediately and blocks in the client.
6357        let first = tokio::spawn({
6358            let ctx = ctx.clone();
6359            async move { ctx.checkpoint_updates(vec![make_update("op-first")]).await }
6360        });
6361        tokio::task::yield_now().await;
6362
6363        // Two more checkpoints arrive while the first write is in flight:
6364        // they join the open batch behind the writer lock.
6365        let second = tokio::spawn({
6366            let ctx = ctx.clone();
6367            async move { ctx.checkpoint_updates(vec![make_update("op-second")]).await }
6368        });
6369        let third = tokio::spawn({
6370            let ctx = ctx.clone();
6371            async move { ctx.checkpoint_updates(vec![make_update("op-third")]).await }
6372        });
6373        // Let both join and block on the writer lock / gate.
6374        tokio::time::sleep(Duration::from_millis(1)).await;
6375
6376        // Release both calls (first write, then the batched follow-up).
6377        client.release_one();
6378        client.release_one();
6379
6380        for task in [first, second, third] {
6381            task.await
6382                .expect("contributor task completes")
6383                .expect("contributor observes success");
6384        }
6385
6386        assert_eq!(
6387            client.call_ids(),
6388            vec![
6389                vec!["op-first".to_owned()],
6390                vec!["op-second".to_owned(), "op-third".to_owned()],
6391            ],
6392            "the two checkpoints that arrived during the in-flight write \
6393             must batch into one follow-up call"
6394        );
6395    }
6396
6397    // Lifecycle-event ownership regression tests
6398    //
6399    // The documented contract (`crate::observability`) guarantees that
6400    // every transition the service records emits its lifecycle event.
6401    // These tests pin the two ways the old contributor-owned emission
6402    // violated that: a contributor dropped after joining a buffered batch,
6403    // and a split batch whose earlier chunk persists before a later chunk
6404    // fails.
6405
6406    /// A `MakeWriter` that captures subscriber output in a shared buffer.
6407    #[derive(Clone)]
6408    struct EventCaptureWriter(Arc<std::sync::Mutex<Vec<u8>>>);
6409
6410    impl std::io::Write for EventCaptureWriter {
6411        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
6412            if let Ok(mut inner) = self.0.lock() {
6413                inner.extend_from_slice(buf);
6414            }
6415            Ok(buf.len())
6416        }
6417        fn flush(&mut self) -> std::io::Result<()> {
6418            Ok(())
6419        }
6420    }
6421
6422    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for EventCaptureWriter {
6423        type Writer = EventCaptureWriter;
6424        fn make_writer(&'a self) -> Self::Writer {
6425            self.clone()
6426        }
6427    }
6428
6429    /// Installs a JSON capture subscriber (all levels, flattened fields,
6430    /// span list included) and returns the shared buffer plus the guard.
6431    fn lifecycle_capture() -> (
6432        Arc<std::sync::Mutex<Vec<u8>>>,
6433        tracing::subscriber::DefaultGuard,
6434    ) {
6435        use tracing_subscriber::layer::SubscriberExt as _;
6436        let buffer = Arc::new(std::sync::Mutex::new(Vec::new()));
6437        let fmt_layer = tracing_subscriber::fmt::layer()
6438            .json()
6439            .flatten_event(true)
6440            .with_span_list(true)
6441            .with_writer(EventCaptureWriter(Arc::clone(&buffer)));
6442        let subscriber = tracing_subscriber::registry().with(fmt_layer);
6443        let guard = test_tracing::install_test_subscriber(subscriber);
6444        (buffer, guard)
6445    }
6446
6447    /// Parses the captured output into the JSON lifecycle-event lines with
6448    /// the given event name, returning each line's parsed JSON.
6449    fn captured_lifecycle_events(
6450        buffer: &Arc<std::sync::Mutex<Vec<u8>>>,
6451        event_name: &str,
6452    ) -> Vec<serde_json::Value> {
6453        let output = buffer.lock().map_or_else(
6454            |_| String::new(),
6455            |b| String::from_utf8_lossy(&b).to_string(),
6456        );
6457        output
6458            .lines()
6459            .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
6460            .filter(|v| {
6461                v.get("message").and_then(serde_json::Value::as_str) == Some(event_name)
6462                    && v.get("target").and_then(serde_json::Value::as_str)
6463                        == Some(crate::observability::TARGET)
6464            })
6465            .collect()
6466    }
6467
6468    /// REGRESSION (event ownership, dropped contributor): a buffered
6469    /// contributor dropped after joining a batch, explicitly supported for
6470    /// `race`/`select_ok` losers, must not take its lifecycle events with
6471    /// it. The flush task persists the update, so the write path must emit
6472    /// the event, inside the span the contributor captured.
6473    ///
6474    /// The scenario retries a few times because `tracing`'s global
6475    /// max-level hint and callsite-interest caches are rebuilt whenever any
6476    /// concurrently running test registers or drops a subscriber, and a
6477    /// rebuild racing this test's `DEBUG`-level emission can drop the event
6478    /// before it reaches this test's (thread-local) subscriber. The
6479    /// persistence invariant is asserted on every attempt; a genuine
6480    /// emission regression fails all attempts.
6481    #[tokio::test(start_paused = true)]
6482    async fn dropped_buffered_contributor_still_emits_events_for_persisted_updates() {
6483        use crate::observability::event_names;
6484
6485        let mut emitted = false;
6486        for _attempt in 0..5 {
6487            let (buffer, _guard) = lifecycle_capture();
6488            let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6489            let ctx = coalescing_ctx(client.clone(), Duration::from_hours(1));
6490
6491            // The contributor polls once inside its own span, far enough
6492            // to capture its events and join the coalescing batch, then is
6493            // dropped, exactly like a lost `race` branch.
6494            {
6495                let span = tracing::info_span!("contributor-span");
6496                let _entered = span.enter();
6497                let mut fut = Box::pin(ctx.checkpoint_updates(vec![make_update("op-dropped")]));
6498                let mut poll_cx = std::task::Context::from_waker(std::task::Waker::noop());
6499                assert!(
6500                    Future::poll(fut.as_mut(), &mut poll_cx).is_pending(),
6501                    "the buffered contributor must be awaiting the batch"
6502                );
6503            } // `fut` dropped here, before any flush.
6504
6505            // The end-of-invocation flush persists the batch anyway.
6506            ctx.flush_pending_checkpoints().await;
6507
6508            let ids: Vec<String> = client
6509                .recorded_updates()
6510                .iter()
6511                .map(|u| u.id.clone())
6512                .collect();
6513            assert_eq!(
6514                ids,
6515                vec!["op-dropped".to_owned()],
6516                "the dropped contributor's update must still be persisted"
6517            );
6518
6519            let events = captured_lifecycle_events(&buffer, event_names::OPERATION_STARTED);
6520            if events.is_empty() {
6521                // A concurrent subscriber rebuild dropped the event before
6522                // it reached this test's subscriber; retry the scenario.
6523                continue;
6524            }
6525
6526            let matching: Vec<&serde_json::Value> = events
6527                .iter()
6528                .filter(|v| {
6529                    v.get(crate::tracing_layer::fields::OPERATION_ID)
6530                        .and_then(serde_json::Value::as_str)
6531                        == Some("op-dropped")
6532                })
6533                .collect();
6534            assert_eq!(
6535                matching.len(),
6536                1,
6537                "the persisted transition must emit exactly one operation_started \
6538                 event even though its contributor was dropped; got {events:?}"
6539            );
6540            let spans = format!("{:?}", matching.first().and_then(|v| v.get("spans")));
6541            assert!(
6542                spans.contains("contributor-span"),
6543                "the event must carry the originating contributor's span context, \
6544                 got spans: {spans}"
6545            );
6546            emitted = true;
6547            break;
6548        }
6549        assert!(
6550            emitted,
6551            "no attempt emitted operation_started for the dropped contributor's \
6552             persisted update — telemetry for a persisted transition was lost"
6553        );
6554    }
6555
6556    /// A client whose second (and any later) `checkpoint` call fails,
6557    /// delegating the first to an [`InMemoryExecutionClient`]. Lets tests
6558    /// persist an early batch chunk and then fail a later one.
6559    #[derive(Debug)]
6560    struct FailSecondCallClient {
6561        inner: InMemoryExecutionClient,
6562        calls: std::sync::Mutex<u32>,
6563    }
6564
6565    impl FailSecondCallClient {
6566        fn new() -> Self {
6567            Self {
6568                inner: InMemoryExecutionClient::new(Vec::new()),
6569                calls: std::sync::Mutex::new(0),
6570            }
6571        }
6572    }
6573
6574    impl ExecutionClient for FailSecondCallClient {
6575        fn checkpoint(
6576            &self,
6577            execution_arn: &str,
6578            checkpoint_token: &str,
6579            updates: Vec<OperationUpdate>,
6580        ) -> std::pin::Pin<
6581            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6582        > {
6583            let arn = execution_arn.to_owned();
6584            let token = checkpoint_token.to_owned();
6585            Box::pin(async move {
6586                let call_index = {
6587                    let mut calls = self
6588                        .calls
6589                        .lock()
6590                        .unwrap_or_else(std::sync::PoisonError::into_inner);
6591                    *calls += 1;
6592                    *calls
6593                };
6594                if call_index >= 2 {
6595                    return Err(ExecutionClientError::new_non_retryable(
6596                        "injected failure on the second chunk",
6597                    ));
6598                }
6599                self.inner.checkpoint(&arn, &token, updates).await
6600            })
6601        }
6602
6603        fn get_state(
6604            &self,
6605            execution_arn: &str,
6606            checkpoint_token: &str,
6607            marker: Option<&str>,
6608        ) -> std::pin::Pin<
6609            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6610        > {
6611            self.inner
6612                .get_state(execution_arn, checkpoint_token, marker)
6613        }
6614    }
6615
6616    /// REGRESSION (event ownership, split batch): when a sealed batch splits
6617    /// into several requests and an earlier request persists before a later
6618    /// one fails, the persisted chunk's events must be emitted: the failure,
6619    /// which leaves every contributor unresolved, must not suppress
6620    /// telemetry for transitions the service actually recorded.
6621    ///
6622    /// Retries for the same reason as
6623    /// [`dropped_buffered_contributor_still_emits_events_for_persisted_updates`]:
6624    /// concurrent tests rebuilding `tracing`'s global caches can race a
6625    /// `DEBUG` emission. Hard invariants assert on every attempt.
6626    #[tokio::test(start_paused = true)]
6627    async fn split_batch_emits_events_for_chunks_persisted_before_a_failure() {
6628        use crate::observability::event_names;
6629
6630        let mut emitted = false;
6631        for _attempt in 0..5 {
6632            let (buffer, _guard) = lifecycle_capture();
6633            let client = Arc::new(FailSecondCallClient::new());
6634            let limits = BatchLimits {
6635                max_operations: 1,
6636                max_payload_bytes: usize::MAX,
6637            };
6638            let ctx = DurableContext::new_root_with_client_and_coalescer(
6639                "arn:test".to_owned(),
6640                lambda_runtime::Context::default(),
6641                Arc::new(CheckpointLog::empty()),
6642                Arc::clone(&client) as Arc<dyn ExecutionClient>,
6643                "token-0".to_owned(),
6644                CheckpointCoalescer::with_limits(Duration::from_millis(50), limits),
6645            );
6646
6647            // Two contributors share one batch; the one-op cap splits it
6648            // into two requests. The first persists, the second fails.
6649            let first = ctx.submit_checkpoint_updates(vec![make_update("op-persisted")]);
6650            let second = ctx.submit_checkpoint_updates(vec![make_update("op-rejected")]);
6651            until_write_failed(&ctx).await;
6652            let fatal = ctx.coordinator().write_failure().expect("latched");
6653            assert_eq!(
6654                fatal.error_type,
6655                crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
6656            );
6657            let both = async { tokio::join!(first, second) };
6658            assert!(
6659                tokio::time::timeout(Duration::from_secs(5), both)
6660                    .await
6661                    .is_err(),
6662                "no contributor resolves"
6663            );
6664
6665            let ids: Vec<String> = client
6666                .inner
6667                .recorded_updates()
6668                .iter()
6669                .map(|u| u.id.clone())
6670                .collect();
6671            assert_eq!(
6672                ids,
6673                vec!["op-persisted".to_owned()],
6674                "exactly the first chunk was persisted"
6675            );
6676
6677            let events = captured_lifecycle_events(&buffer, event_names::OPERATION_STARTED);
6678            if events.is_empty() {
6679                // A concurrent subscriber rebuild dropped the event before
6680                // it reached this test's subscriber; retry the scenario.
6681                continue;
6682            }
6683
6684            let persisted: Vec<&serde_json::Value> = events
6685                .iter()
6686                .filter(|v| {
6687                    v.get(crate::tracing_layer::fields::OPERATION_ID)
6688                        .and_then(serde_json::Value::as_str)
6689                        == Some("op-persisted")
6690                })
6691                .collect();
6692            assert_eq!(
6693                persisted.len(),
6694                1,
6695                "the persisted chunk must emit its operation_started event \
6696                 despite the later chunk's failure; got {events:?}"
6697            );
6698            assert!(
6699                !events.iter().any(|v| {
6700                    v.get(crate::tracing_layer::fields::OPERATION_ID)
6701                        .and_then(serde_json::Value::as_str)
6702                        == Some("op-rejected")
6703                }),
6704                "the rejected chunk recorded nothing, so it must emit nothing; \
6705                 got {events:?}"
6706            );
6707            emitted = true;
6708            break;
6709        }
6710        assert!(
6711            emitted,
6712            "no attempt emitted operation_started for the persisted chunk — \
6713             telemetry for a persisted transition was lost"
6714        );
6715    }
6716
6717    /// REGRESSION (event ownership, failed pagination hydration): the
6718    /// checkpoint call can persist the transitions and *then* fail while
6719    /// hydrating paginated state through `get_state`. The persisted
6720    /// transitions' events must already be emitted by then, emission
6721    /// happens immediately after the service accepts the write, before the
6722    /// fallible pagination fetch, even though the write fails.
6723    ///
6724    /// Retries for the same reason as
6725    /// [`dropped_buffered_contributor_still_emits_events_for_persisted_updates`]:
6726    /// concurrent tests rebuilding `tracing`'s global caches can race a
6727    /// `DEBUG` emission. Hard invariants assert on every attempt.
6728    #[tokio::test]
6729    async fn persisted_checkpoint_emits_events_when_pagination_hydration_fails() {
6730        use crate::observability::event_names;
6731
6732        let mut emitted = false;
6733        for _attempt in 0..5 {
6734            let (buffer, _guard) = lifecycle_capture();
6735            let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6736            // The write itself succeeds but signals more pages, and the
6737            // follow-up get_state fetch fails.
6738            client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
6739                Vec::new(),
6740                "page-2-token".to_owned(),
6741            ));
6742            client.fail_get_state("injected get_state failure");
6743
6744            let ctx = DurableContext::new_root_with_client(
6745                "arn:test".to_owned(),
6746                lambda_runtime::Context::default(),
6747                Arc::new(CheckpointLog::empty()),
6748                Arc::clone(&client) as Arc<dyn ExecutionClient>,
6749                "token-0".to_owned(),
6750            );
6751
6752            let write = ctx.submit_checkpoint_updates(vec![make_update("op-hydration")]);
6753            until_write_failed(&ctx).await;
6754            let fatal = ctx.coordinator().write_failure().expect("latched");
6755            assert_eq!(
6756                fatal.error_type,
6757                crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
6758            );
6759            assert!(
6760                tokio::time::timeout(Duration::from_millis(100), write)
6761                    .await
6762                    .is_err(),
6763                "the write's caller never resolves"
6764            );
6765            let ids: Vec<String> = client
6766                .recorded_updates()
6767                .iter()
6768                .map(|u| u.id.clone())
6769                .collect();
6770            assert_eq!(
6771                ids,
6772                vec!["op-hydration".to_owned()],
6773                "the transition was persisted before get_state failed"
6774            );
6775
6776            let events = captured_lifecycle_events(&buffer, event_names::OPERATION_STARTED);
6777            if events.is_empty() {
6778                // A concurrent subscriber rebuild dropped the event before
6779                // it reached this test's subscriber; retry the scenario.
6780                continue;
6781            }
6782
6783            let matching: Vec<&serde_json::Value> = events
6784                .iter()
6785                .filter(|v| {
6786                    v.get(crate::tracing_layer::fields::OPERATION_ID)
6787                        .and_then(serde_json::Value::as_str)
6788                        == Some("op-hydration")
6789                })
6790                .collect();
6791            assert_eq!(
6792                matching.len(),
6793                1,
6794                "the persisted transition must emit exactly one operation_started \
6795                 event even though pagination hydration failed; got {events:?}"
6796            );
6797            emitted = true;
6798            break;
6799        }
6800        assert!(
6801            emitted,
6802            "no attempt emitted operation_started for the persisted transition — \
6803             telemetry was suppressed by a failed pagination fetch"
6804        );
6805    }
6806
6807    /// Waits, for at most five seconds, until a write of `ctx` failed.
6808    async fn until_write_failed(ctx: &DurableContext) {
6809        tokio::time::timeout(Duration::from_secs(5), async {
6810            while !ctx.coordinator().write_failed() {
6811                tokio::time::sleep(Duration::from_millis(1)).await;
6812            }
6813        })
6814        .await
6815        .expect("a write failed");
6816    }
6817
6818    // Checkpoint admission tests (abandoned owners).
6819
6820    /// The bound on every await in the admission tests. A rejected write
6821    /// never resolves, so these awaits time out.
6822    const ADMISSION_BOUND: Duration = Duration::from_secs(1);
6823
6824    /// A root context with the coalescing window `window`.
6825    fn admission_root(client: &Arc<InMemoryExecutionClient>, window: Duration) -> DurableContext {
6826        DurableContext::new_root_with_client_and_defaults(
6827            "arn:test".to_owned(),
6828            lambda_runtime::Context::default(),
6829            Arc::new(CheckpointLog::empty()),
6830            Arc::clone(client) as Arc<dyn ExecutionClient>,
6831            "token-0".to_owned(),
6832            window,
6833            0,
6834            None,
6835        )
6836    }
6837
6838    /// The wire IDs the client recorded, in order.
6839    fn recorded_ids(client: &InMemoryExecutionClient) -> Vec<String> {
6840        client
6841            .recorded_updates()
6842            .iter()
6843            .map(|u| u.id.clone())
6844            .collect()
6845    }
6846
6847    /// A write from an abandoned item never resolves, and nothing is
6848    /// recorded.
6849    #[tokio::test]
6850    async fn abandoned_owner_update_is_not_sent() {
6851        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6852        let root = admission_root(&client, Duration::ZERO);
6853        let item = root.new_child("1", None);
6854        root.admission()
6855            .abandon(item.owners().iter().map(String::as_str));
6856        let written = tokio::time::timeout(
6857            ADMISSION_BOUND,
6858            item.checkpoint_updates(vec![make_update("u")]),
6859        )
6860        .await;
6861        assert!(written.is_err(), "a rejected write never resolves");
6862        assert!(client.recorded_updates().is_empty(), "nothing is sent");
6863    }
6864
6865    #[tokio::test]
6866    async fn nested_owner_follows_its_outer_item() {
6867        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6868        let root = admission_root(&client, Duration::ZERO);
6869        let item = root.new_child("1", None);
6870        let inner = item.new_child("1-1", None);
6871        assert_eq!(
6872            inner.owners(),
6873            [
6874                crate::engine::compute_wire_id_public("1"),
6875                crate::engine::compute_wire_id_public("1-1"),
6876            ],
6877            "a nested context runs in its outer item, outermost first"
6878        );
6879        root.admission()
6880            .abandon([crate::engine::compute_wire_id_public("1").as_str()]);
6881        let written = tokio::time::timeout(
6882            ADMISSION_BOUND,
6883            inner.checkpoint_updates(vec![make_update("u")]),
6884        )
6885        .await;
6886        assert!(written.is_err(), "the nested write never resolves");
6887        assert!(client.recorded_updates().is_empty(), "nothing is sent");
6888    }
6889
6890    #[tokio::test(start_paused = true)]
6891    async fn admitted_update_precedes_parent_final_update() {
6892        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
6893        let root = admission_root(&client, Duration::from_millis(50));
6894        let item = root.new_child("1", None);
6895        let writer = tokio::spawn({
6896            let item = item.clone();
6897            async move { item.checkpoint_updates(vec![make_update("a")]).await }
6898        });
6899        // Yield until `a` is buffered. The delay window holds it there, so
6900        // it is admitted but not yet sent when the item is abandoned.
6901        let coalescer = Arc::clone(&root.inner.coalescer);
6902        let mut yields = 0;
6903        while coalescer.pending_len() == 0 {
6904            assert!(yields < 1_000, "the item update was never buffered");
6905            yields += 1;
6906            tokio::task::yield_now().await;
6907        }
6908        root.admission()
6909            .abandon(item.owners().iter().map(String::as_str));
6910        let parent = tokio::spawn({
6911            let root = root.clone();
6912            async move { root.checkpoint_updates(vec![make_update("p")]).await }
6913        });
6914        tokio::time::advance(Duration::from_millis(100)).await;
6915        tokio::time::timeout(ADMISSION_BOUND, writer)
6916            .await
6917            .expect("the admitted write resolves")
6918            .expect("the writer task completes")
6919            .expect("the admitted write succeeds");
6920        tokio::time::timeout(ADMISSION_BOUND, parent)
6921            .await
6922            .expect("the parent write resolves")
6923            .expect("the parent task completes")
6924            .expect("the parent write succeeds");
6925        assert_eq!(
6926            recorded_ids(&client),
6927            ["a", "p"],
6928            "the admitted item update reaches the service before the parent's"
6929        );
6930    }
6931
6932    /// A client that holds its first checkpoint call until the test opens a
6933    /// gate, and records the token and the update IDs of every call. Call
6934    /// `n` returns the token `token-n`.
6935    #[derive(Debug, Default)]
6936    struct TokenGateClient {
6937        entered: tokio::sync::Notify,
6938        gate: tokio::sync::Notify,
6939        calls: std::sync::Mutex<Vec<(String, Vec<String>)>>,
6940    }
6941
6942    impl TokenGateClient {
6943        fn calls(&self) -> Vec<(String, Vec<String>)> {
6944            self.calls.lock().unwrap().clone()
6945        }
6946    }
6947
6948    impl ExecutionClient for TokenGateClient {
6949        fn checkpoint(
6950            &self,
6951            _arn: &str,
6952            token: &str,
6953            updates: Vec<OperationUpdate>,
6954        ) -> std::pin::Pin<
6955            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
6956        > {
6957            let token = token.to_owned();
6958            Box::pin(async move {
6959                let n = {
6960                    let mut calls = self.calls.lock().unwrap();
6961                    calls.push((token, updates.iter().map(|u| u.id.clone()).collect()));
6962                    calls.len()
6963                };
6964                if n == 1 {
6965                    self.entered.notify_one();
6966                    self.gate.notified().await;
6967                }
6968                Ok(CheckpointOutput {
6969                    checkpoint_token: Some(format!("token-{n}")),
6970                    updated_operations: Vec::new(),
6971                    next_marker: None,
6972                })
6973            })
6974        }
6975
6976        fn get_state(
6977            &self,
6978            _arn: &str,
6979            _token: &str,
6980            _marker: Option<&str>,
6981        ) -> std::pin::Pin<
6982            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
6983        > {
6984            Box::pin(async {
6985                Ok(GetStateOutput {
6986                    operations: Vec::new(),
6987                    next_marker: None,
6988                })
6989            })
6990        }
6991    }
6992
6993    #[tokio::test]
6994    async fn admitted_write_survives_cancelled_caller() {
6995        let client = Arc::new(TokenGateClient::default());
6996        let root = DurableContext::new_root_with_client_and_defaults(
6997            "arn:test".to_owned(),
6998            lambda_runtime::Context::default(),
6999            Arc::new(CheckpointLog::empty()),
7000            Arc::clone(&client) as Arc<dyn ExecutionClient>,
7001            "token-0".to_owned(),
7002            Duration::ZERO,
7003            0,
7004            None,
7005        );
7006        let item = root.new_child("1", None);
7007        let entered = client.entered.notified();
7008        let writer = tokio::spawn({
7009            let item = item.clone();
7010            async move { item.checkpoint_updates(vec![make_update("a")]).await }
7011        });
7012        tokio::time::timeout(ADMISSION_BOUND, entered)
7013            .await
7014            .expect("the client holds the item's write");
7015        writer.abort();
7016        assert!(
7017            writer
7018                .await
7019                .expect_err("the caller was aborted")
7020                .is_cancelled(),
7021            "the caller is cancelled while its write is in flight"
7022        );
7023        client.gate.notify_one();
7024        tokio::time::timeout(
7025            ADMISSION_BOUND,
7026            root.checkpoint_updates(vec![make_update("p")]),
7027        )
7028        .await
7029        .expect("the parent write resolves")
7030        .expect("the parent write succeeds");
7031        assert_eq!(
7032            client.calls(),
7033            [
7034                ("token-0".to_owned(), vec!["a".to_owned()]),
7035                ("token-1".to_owned(), vec!["p".to_owned()]),
7036            ],
7037            "the cancelled caller's write completed and rotated the token"
7038        );
7039    }
7040
7041    #[tokio::test]
7042    async fn abandoned_owner_starts_no_operation() {
7043        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7044        let root = admission_root(&client, Duration::ZERO);
7045        let item = root.new_child("1", None);
7046        root.admission()
7047            .abandon(item.owners().iter().map(String::as_str));
7048        let ran = Arc::new(std::sync::atomic::AtomicBool::new(false));
7049        let flag = Arc::clone(&ran);
7050        let stepped = tokio::time::timeout(
7051            ADMISSION_BOUND,
7052            item.step(move |_| async move {
7053                flag.store(true, std::sync::atomic::Ordering::SeqCst);
7054                Ok(1_u32)
7055            })
7056            .into_future(),
7057        )
7058        .await;
7059        assert!(stepped.is_err(), "the operation never resolves");
7060        assert!(
7061            !ran.load(std::sync::atomic::Ordering::SeqCst),
7062            "the step body never ran"
7063        );
7064        assert!(
7065            client.recorded_updates().is_empty(),
7066            "no update is recorded"
7067        );
7068    }
7069
7070    /// An abandoned owner does not even replay a recorded operation, which
7071    /// writes nothing: the builder future is never polled past the check.
7072    #[tokio::test]
7073    async fn abandoned_owner_replays_no_operation() {
7074        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7075        let wire = crate::engine::compute_wire_id_public("1-1");
7076        let record = CheckpointRecord {
7077            id: wire.clone(),
7078            status: CheckpointStatus::Succeeded,
7079            result: Some("7".to_owned()),
7080            error_type: None,
7081            error_message: None,
7082            error_data: None,
7083            stack_trace: None,
7084            attempt: 0,
7085            invoke_result: None,
7086            invoke_error_type: None,
7087            invoke_error_message: None,
7088            invoke_error_data: None,
7089            invoke_stack_trace: None,
7090            replay_children: false,
7091            callback_id: None,
7092            op_type: None,
7093            sub_type: None,
7094            due_at: None,
7095            op_name: None,
7096            parent_id: None,
7097            start_timestamp: None,
7098            end_timestamp: None,
7099        };
7100        let log = CheckpointLog::from_records(vec![(wire, record)]);
7101        let root = DurableContext::new_root_with_client_and_defaults(
7102            "arn:test".to_owned(),
7103            lambda_runtime::Context::default(),
7104            Arc::new(log),
7105            Arc::clone(&client) as Arc<dyn ExecutionClient>,
7106            "token-0".to_owned(),
7107            Duration::ZERO,
7108            0,
7109            None,
7110        );
7111        let item = root.new_child("1", None);
7112        root.admission()
7113            .abandon(item.owners().iter().map(String::as_str));
7114        let stepped = tokio::time::timeout(
7115            ADMISSION_BOUND,
7116            item.step(|_| async { Ok(0_u32) }).into_future(),
7117        )
7118        .await;
7119        assert!(
7120            stepped.is_err(),
7121            "the recorded step is not replayed for an abandoned owner: {stepped:?}"
7122        );
7123    }
7124
7125    /// A builder created before its owner is abandoned, and finalized after,
7126    /// is not checked against a contradicting record. The record says the
7127    /// operation was a wait, and the builder claims a step. For an owner that
7128    /// is not abandoned, the same claim records a fatal error and resolves.
7129    /// For the abandoned owner, nothing is recorded, nothing is written, and
7130    /// the future never resolves.
7131    #[tokio::test]
7132    async fn abandoned_owner_skips_replay_preflight() {
7133        let wait_record = |positional: &str| {
7134            let wire = crate::engine::compute_wire_id_public(positional);
7135            let record = CheckpointRecord {
7136                id: wire.clone(),
7137                status: CheckpointStatus::Succeeded,
7138                result: None,
7139                error_type: None,
7140                error_message: None,
7141                error_data: None,
7142                stack_trace: None,
7143                attempt: 0,
7144                invoke_result: None,
7145                invoke_error_type: None,
7146                invoke_error_message: None,
7147                invoke_error_data: None,
7148                invoke_stack_trace: None,
7149                replay_children: false,
7150                callback_id: None,
7151                op_type: Some("Wait".to_owned()),
7152                sub_type: Some("Wait".to_owned()),
7153                due_at: None,
7154                op_name: None,
7155                parent_id: None,
7156                start_timestamp: None,
7157                end_timestamp: None,
7158            };
7159            (wire, record)
7160        };
7161        let build_root = || {
7162            let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7163            let log = CheckpointLog::from_records(vec![wait_record("1-1")]);
7164            let root = DurableContext::new_root_with_client_and_defaults(
7165                "arn:test".to_owned(),
7166                lambda_runtime::Context::default(),
7167                Arc::new(log),
7168                Arc::clone(&client) as Arc<dyn ExecutionClient>,
7169                "token-0".to_owned(),
7170                Duration::ZERO,
7171                0,
7172                None,
7173            );
7174            (client, root)
7175        };
7176
7177        // Control: an owner that is not abandoned records the mismatch.
7178        let (_, control_root) = build_root();
7179        let control_item = control_root.new_child("1", None);
7180        let control = tokio::time::timeout(
7181            ADMISSION_BOUND,
7182            control_item.step(|_| async { Ok(0_u32) }).into_future(),
7183        )
7184        .await
7185        .expect("the mismatch resolves at once");
7186        assert!(control.is_err(), "the mismatch is an error: {control:?}");
7187        assert!(
7188            control_item.coordinator().fatal_error().is_some(),
7189            "the mismatch records a fatal error"
7190        );
7191
7192        // The builder exists before the abandonment and is finalized after it.
7193        let (client, root) = build_root();
7194        let item = root.new_child("1", None);
7195        let builder = item.step(|_| async { Ok(0_u32) });
7196        root.admission()
7197            .abandon(item.owners().iter().map(String::as_str));
7198        let stepped = tokio::time::timeout(ADMISSION_BOUND, builder.into_future()).await;
7199        assert!(
7200            stepped.is_err(),
7201            "the abandoned owner's future never resolves: {stepped:?}"
7202        );
7203        assert!(
7204            item.coordinator().fatal_error().is_none(),
7205            "no fatal error is recorded for the abandoned owner"
7206        );
7207        assert!(
7208            root.coordinator().fatal_error().is_none(),
7209            "no fatal error reaches the root"
7210        );
7211        assert!(client.recorded_updates().is_empty(), "nothing is written");
7212    }
7213
7214    /// An abandoned owner does not replay a recorded callback either. The
7215    /// recorded `Started` callback resolves without a write, so only the
7216    /// admission check at the terminal can hold it back.
7217    #[tokio::test]
7218    async fn abandoned_owner_replays_no_callback() {
7219        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7220        let wire = crate::engine::compute_wire_id_public("1-1");
7221        let record = CheckpointRecord {
7222            id: wire.clone(),
7223            status: CheckpointStatus::Started,
7224            result: None,
7225            error_type: None,
7226            error_message: None,
7227            error_data: None,
7228            stack_trace: None,
7229            attempt: 0,
7230            invoke_result: None,
7231            invoke_error_type: None,
7232            invoke_error_message: None,
7233            invoke_error_data: None,
7234            invoke_stack_trace: None,
7235            replay_children: false,
7236            callback_id: Some("cb-1".to_owned()),
7237            op_type: None,
7238            sub_type: None,
7239            due_at: None,
7240            op_name: None,
7241            parent_id: None,
7242            start_timestamp: None,
7243            end_timestamp: None,
7244        };
7245        let log = Arc::new(CheckpointLog::from_records(vec![(wire, record)]));
7246        let root = DurableContext::new_root_with_client_and_defaults(
7247            "arn:test".to_owned(),
7248            lambda_runtime::Context::default(),
7249            Arc::clone(&log),
7250            Arc::clone(&client) as Arc<dyn ExecutionClient>,
7251            "token-0".to_owned(),
7252            Duration::ZERO,
7253            0,
7254            None,
7255        );
7256        // The control: an owner that is not abandoned replays the callback.
7257        let control = root.new_child("1", None);
7258        let replayed = tokio::time::timeout(
7259            ADMISSION_BOUND,
7260            control.create_callback::<String>().into_future(),
7261        )
7262        .await
7263        .expect("the recorded callback replays")
7264        .expect("the replay succeeds");
7265        assert_eq!(replayed.id(), "cb-1");
7266
7267        let item = root.new_child("1", None);
7268        root.admission()
7269            .abandon(item.owners().iter().map(String::as_str));
7270        let created = tokio::time::timeout(
7271            ADMISSION_BOUND,
7272            item.create_callback::<String>().into_future(),
7273        )
7274        .await;
7275        assert!(
7276            created.is_err(),
7277            "the recorded callback is not replayed for an abandoned owner"
7278        );
7279        assert!(client.recorded_updates().is_empty(), "nothing is sent");
7280    }
7281
7282    #[tokio::test]
7283    async fn admitted_owner_writes_as_before() {
7284        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7285        let root = admission_root(&client, Duration::ZERO);
7286        let item = root.new_child("1", None);
7287        tokio::time::timeout(
7288            ADMISSION_BOUND,
7289            item.checkpoint_updates(vec![make_update("u")]),
7290        )
7291        .await
7292        .expect("the write resolves")
7293        .expect("the write succeeds");
7294        assert_eq!(recorded_ids(&client), ["u"]);
7295    }
7296
7297    /// Without a Tokio runtime a write cannot start its task. The
7298    /// caller gets the no-runtime error at once, and nothing is sent.
7299    #[test]
7300    fn write_without_runtime_returns_invocation_error() {
7301        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7302        let ctx = admission_root(&client, Duration::ZERO);
7303        let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7304            ctx.checkpoint_updates(vec![make_update("u")]).await
7305        });
7306        let std::task::Poll::Ready(Err(err)) = polled else {
7307            unreachable!("expected an immediate error, got {polled:?}");
7308        };
7309        assert_eq!(err.scope(), ExecutionClientErrorScope::Invocation);
7310        assert!(
7311            err.to_string().contains(crate::task::NO_RUNTIME_MESSAGE),
7312            "unexpected error: {err}"
7313        );
7314        assert!(
7315            std::error::Error::source(&err)
7316                .is_some_and(<dyn std::error::Error>::is::<crate::task::NoRuntime>),
7317            "the error carries the no-runtime marker"
7318        );
7319        assert!(client.recorded_updates().is_empty(), "nothing is sent");
7320    }
7321
7322    /// An operation whose write finds no Tokio runtime returns the
7323    /// error `.spawn()` returns without a runtime, at once. Nothing is
7324    /// sent, and no invocation fault is recorded.
7325    #[test]
7326    fn operation_without_runtime_returns_spawn_error() {
7327        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7328        let thread_client = Arc::clone(&client);
7329        // The context is built on the polling thread, so that thread owns it.
7330        let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7331            let ctx = admission_root(&thread_client, Duration::ZERO);
7332            let signal = Arc::clone(ctx.coordinator());
7333            (
7334                ctx.step(|_| async { Ok(1_u32) }).into_future().await,
7335                signal,
7336            )
7337        });
7338        let std::task::Poll::Ready((Err(err), signal)) = polled else {
7339            unreachable!("expected an immediate error, got {polled:?}");
7340        };
7341        let spawn_error = crate::task::NoRuntime::operation_error(crate::task::NoRuntime);
7342        assert_eq!(format!("{err:#}"), format!("{spawn_error:#}"));
7343        assert!(client.recorded_updates().is_empty(), "nothing is sent");
7344        assert!(signal.fatal_error().is_none(), "no fault is recorded");
7345    }
7346
7347    /// An operation whose buffered write finds no Tokio runtime returns
7348    /// the error `.spawn()` returns without a runtime, at once: the batch
7349    /// flusher cannot be spawned. Nothing is sent, and no fatal error is
7350    /// recorded.
7351    #[test]
7352    fn buffered_write_without_runtime_returns_the_no_runtime_error() {
7353        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7354        let thread_client = Arc::clone(&client);
7355        // The context is built on the polling thread, so that thread owns it.
7356        let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7357            let ctx = admission_root(&thread_client, Duration::ZERO);
7358            (ctx.step(|_| async { Ok(1_u32) }).into_future().await, ctx)
7359        });
7360        let std::task::Poll::Ready((Err(err), ctx)) = polled else {
7361            unreachable!("expected an immediate error, got {polled:?}");
7362        };
7363        let spawn_error = crate::task::NoRuntime::operation_error(crate::task::NoRuntime);
7364        assert_eq!(format!("{err:#}"), format!("{spawn_error:#}"));
7365        assert!(client.recorded_updates().is_empty(), "nothing is sent");
7366        assert!(
7367            ctx.coordinator().fatal_error().is_none(),
7368            "no fatal error is recorded"
7369        );
7370        // The failed write left the buffer, so a flush under a runtime
7371        // sends nothing.
7372        assert!(ctx.queued_update_ids().is_empty(), "the batch is dropped");
7373        let runtime = tokio::runtime::Builder::new_current_thread()
7374            .enable_all()
7375            .build()
7376            .unwrap();
7377        runtime.block_on(ctx.flush_pending_checkpoints());
7378        assert!(
7379            client.recorded_updates().is_empty(),
7380            "the dropped batch is never sent"
7381        );
7382        assert!(ctx.coordinator().fatal_error().is_none());
7383    }
7384
7385    /// After a failed write, a write on a thread without a Tokio runtime is
7386    /// rejected before its flusher is spawned: it never resolves, and
7387    /// nothing is sent.
7388    #[test]
7389    fn write_after_failure_without_runtime_never_resolves() {
7390        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7391        let thread_client = Arc::clone(&client);
7392        let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7393            let ctx = admission_root(&thread_client, Duration::ZERO);
7394            ctx.coordinator().fail_write(
7395                FatalError {
7396                    error_type: "Failed".to_owned(),
7397                    error_message: "failed".to_owned(),
7398                    severity: FatalSeverity::FailExecution,
7399                },
7400                || {},
7401            );
7402            ctx.checkpoint_updates(vec![make_update("u")]).await
7403        });
7404        assert!(polled.is_pending(), "{polled:?}");
7405        assert!(client.recorded_updates().is_empty(), "nothing is sent");
7406    }
7407
7408    /// A write on a thread without a Tokio runtime, whose flusher cannot be
7409    /// spawned, meets a failed write after it joined the batch and before
7410    /// it hands out the `NoRuntime` error: the thread sets the latch when it
7411    /// reaches that point, and the failure does not drop the batch. The failure came first, so the write is
7412    /// dropped unresolved, as the failure drops every queued write, and
7413    /// nothing is sent.
7414    #[test]
7415    fn no_runtime_error_after_a_failed_write_is_not_published() {
7416        use crate::coordinator::race_points::{self, RacePoint};
7417        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7418        let thread_client = Arc::clone(&client);
7419        let fired = Arc::new(std::sync::atomic::AtomicBool::new(false));
7420        let thread_fired = Arc::clone(&fired);
7421        let polled = crate::task::test_support::poll_once_without_runtime(move || {
7422            let ctx = admission_root(&thread_client, Duration::ZERO);
7423            let failing = ctx.clone();
7424            race_points::set(move |reached| {
7425                if reached == RacePoint::BatchNoRuntimeClaim {
7426                    thread_fired.store(true, std::sync::atomic::Ordering::SeqCst);
7427                    failing.coordinator().fail_write(
7428                        FatalError {
7429                            error_type: "Failed".to_owned(),
7430                            error_message: "failed".to_owned(),
7431                            severity: FatalSeverity::FailExecution,
7432                        },
7433                        || {},
7434                    );
7435                }
7436            });
7437            async move { ctx.checkpoint_updates(vec![make_update("u")]).await }
7438        });
7439        assert!(
7440            fired.load(std::sync::atomic::Ordering::SeqCst),
7441            "the point is reached"
7442        );
7443        assert!(polled.is_pending(), "{polled:?}");
7444        assert!(client.recorded_updates().is_empty(), "nothing is sent");
7445    }
7446
7447    /// A replayed `Started` child skips its START write, so its body runs
7448    /// and its first write is the terminal `FAIL`. When that write finds
7449    /// no Tokio runtime, the child returns the error `.spawn()`
7450    /// returns without a runtime. Nothing is sent, and no invocation fault
7451    /// is recorded.
7452    #[test]
7453    fn replayed_child_failure_without_runtime_returns_spawn_error() {
7454        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7455        let wire = crate::engine::compute_wire_id_public("1");
7456        let record = CheckpointRecord {
7457            id: wire.clone(),
7458            status: CheckpointStatus::Started,
7459            result: None,
7460            error_type: None,
7461            error_message: None,
7462            error_data: None,
7463            stack_trace: None,
7464            attempt: 0,
7465            invoke_result: None,
7466            invoke_error_type: None,
7467            invoke_error_message: None,
7468            invoke_error_data: None,
7469            invoke_stack_trace: None,
7470            replay_children: false,
7471            callback_id: None,
7472            op_type: Some("Context".to_owned()),
7473            sub_type: Some(crate::child::CHILD_SUB_TYPE.to_owned()),
7474            due_at: None,
7475            op_name: None,
7476            parent_id: None,
7477            start_timestamp: None,
7478            end_timestamp: None,
7479        };
7480        let thread_client = Arc::clone(&client);
7481        // The context is built on the polling thread, so that thread owns it.
7482        let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
7483            let ctx = DurableContext::new_root_with_client_and_defaults(
7484                "arn:test".to_owned(),
7485                lambda_runtime::Context::default(),
7486                Arc::new(CheckpointLog::from_records(vec![(wire, record)])),
7487                thread_client as Arc<dyn ExecutionClient>,
7488                "token-0".to_owned(),
7489                Duration::ZERO,
7490                0,
7491                None,
7492            );
7493            let signal = Arc::clone(ctx.coordinator());
7494            let outcome = ctx
7495                .run_in_child_context(|_child| async { Err::<u32, BoxError>("boom".into()) })
7496                .into_future()
7497                .await;
7498            (outcome, signal)
7499        });
7500        let std::task::Poll::Ready((Err(err), signal)) = polled else {
7501            unreachable!("expected an immediate error, got {polled:?}");
7502        };
7503        let spawn_error = crate::task::NoRuntime::operation_error(crate::task::NoRuntime);
7504        assert_eq!(format!("{err:#}"), format!("{spawn_error:#}"));
7505        assert!(client.recorded_updates().is_empty(), "nothing is sent");
7506        assert!(signal.fatal_error().is_none(), "no fault is recorded");
7507    }
7508
7509    /// A client whose checkpoint call panics.
7510    #[derive(Debug)]
7511    struct PanickingClient;
7512
7513    impl ExecutionClient for PanickingClient {
7514        fn checkpoint(
7515            &self,
7516            _arn: &str,
7517            _token: &str,
7518            _updates: Vec<OperationUpdate>,
7519        ) -> std::pin::Pin<
7520            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
7521        > {
7522            #[expect(clippy::panic)] // reason: the panic is the behavior under test
7523            Box::pin(async { panic!("boom in checkpoint") })
7524        }
7525
7526        fn get_state(
7527            &self,
7528            _arn: &str,
7529            _token: &str,
7530            _marker: Option<&str>,
7531        ) -> std::pin::Pin<
7532            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
7533        > {
7534            Box::pin(async {
7535                Ok(GetStateOutput {
7536                    operations: Vec::new(),
7537                    next_marker: None,
7538                })
7539            })
7540        }
7541    }
7542
7543    /// Writes one update through `client`, and returns the coordinator's
7544    /// kept panic after the flush task stopped. The caller must stay
7545    /// pending: the batch is never published.
7546    async fn write_kept_panic(
7547        client: Arc<dyn ExecutionClient>,
7548    ) -> Box<dyn std::any::Any + Send + 'static> {
7549        let root = DurableContext::new_root_with_client_and_defaults(
7550            "arn:test".to_owned(),
7551            lambda_runtime::Context::default(),
7552            Arc::new(CheckpointLog::empty()),
7553            client,
7554            "token-0".to_owned(),
7555            Duration::ZERO,
7556            0,
7557            None,
7558        );
7559        let coordinator = Arc::clone(root.coordinator());
7560        let mut write = std::pin::pin!(root.checkpoint_updates(vec![make_update("u")]));
7561        let waited = tokio::time::timeout(ADMISSION_BOUND, async {
7562            tokio::select! {
7563                _ = &mut write => false,
7564                () = async {
7565                    // Wait until the write task has stopped, then poll the
7566                    // caller once more.
7567                    tokio::task::yield_now().await;
7568                    coordinator.tasks().wait_idle().await;
7569                } => true,
7570            }
7571        })
7572        .await
7573        .expect("the write task stops");
7574        assert!(waited, "the caller resolved");
7575        let polled =
7576            std::future::poll_fn(|cx| std::task::Poll::Ready(write.as_mut().poll(cx))).await;
7577        assert!(
7578            polled.is_pending(),
7579            "the caller stays pending after the client panicked"
7580        );
7581        coordinator.take_panic().expect("the panic is kept")
7582    }
7583
7584    /// An error source whose destructor panics with "source drop".
7585    #[derive(Debug)]
7586    struct DropPanickingSource;
7587
7588    impl std::fmt::Display for DropPanickingSource {
7589        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
7590            f.write_str("drop-panicking source")
7591        }
7592    }
7593
7594    impl std::error::Error for DropPanickingSource {}
7595
7596    impl Drop for DropPanickingSource {
7597        #[expect(clippy::panic)] // reason: the panic is the behavior under test
7598        fn drop(&mut self) {
7599            panic!("source drop");
7600        }
7601    }
7602
7603    /// A client whose checkpoint call waits for `release`, then fails with
7604    /// an error whose source panics when dropped.
7605    #[derive(Debug)]
7606    struct GatedFailingClient {
7607        release: tokio::sync::Notify,
7608    }
7609
7610    impl ExecutionClient for GatedFailingClient {
7611        fn checkpoint(
7612            &self,
7613            _arn: &str,
7614            _token: &str,
7615            _updates: Vec<OperationUpdate>,
7616        ) -> std::pin::Pin<
7617            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
7618        > {
7619            Box::pin(async {
7620                self.release.notified().await;
7621                Err(ExecutionClientError::with_source(
7622                    ExecutionClientErrorScope::Invocation,
7623                    "gated failure",
7624                    DropPanickingSource,
7625                ))
7626            })
7627        }
7628
7629        fn get_state(
7630            &self,
7631            _arn: &str,
7632            _token: &str,
7633            _marker: Option<&str>,
7634        ) -> std::pin::Pin<
7635            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
7636        > {
7637            Box::pin(async {
7638                Ok(GetStateOutput {
7639                    operations: Vec::new(),
7640                    next_marker: None,
7641                })
7642            })
7643        }
7644    }
7645
7646    /// The caller of a write is dropped before the write returns,
7647    /// and the write fails. The write task drops the client's error, and
7648    /// the task gateway records the panic of its destructor on the
7649    /// coordinator.
7650    #[tokio::test]
7651    async fn unread_write_result_drop_panic_is_recorded() {
7652        let client = Arc::new(GatedFailingClient {
7653            release: tokio::sync::Notify::new(),
7654        });
7655        let root = DurableContext::new_root_with_client_and_defaults(
7656            "arn:test".to_owned(),
7657            lambda_runtime::Context::default(),
7658            Arc::new(CheckpointLog::empty()),
7659            Arc::clone(&client) as Arc<dyn ExecutionClient>,
7660            "token-0".to_owned(),
7661            Duration::ZERO,
7662            0,
7663            None,
7664        );
7665        let coordinator = Arc::clone(root.coordinator());
7666        let mut write = Box::pin(root.checkpoint_updates(vec![make_update("u")]));
7667        let polled =
7668            std::future::poll_fn(|cx| std::task::Poll::Ready(write.as_mut().poll(cx))).await;
7669        assert!(polled.is_pending());
7670        drop(write);
7671        client.release.notify_one();
7672        tokio::time::timeout(ADMISSION_BOUND, coordinator.tasks().wait_idle())
7673            .await
7674            .expect("the write task stops");
7675        let payload = coordinator
7676            .take_panic()
7677            .expect("the destructor's panic is recorded");
7678        assert_eq!(
7679            crate::task::test_support::str_payload(payload.as_ref()),
7680            Some("source drop")
7681        );
7682    }
7683
7684    /// A panic of the checkpoint client in a write's task is kept
7685    /// by the coordinator with its original payload, and the caller stays
7686    /// pending.
7687    #[tokio::test]
7688    async fn write_panic_is_recorded_and_caller_stays_pending() {
7689        let payload = write_kept_panic(Arc::new(PanickingClient)).await;
7690        assert_eq!(
7691            crate::task::test_support::str_payload(payload.as_ref()),
7692            Some("boom in checkpoint")
7693        );
7694    }
7695
7696    /// A checkpoint future that returns `Ready` and then panics in its
7697    /// destructor.
7698    struct ReadyThenDropPanic;
7699
7700    impl Future for ReadyThenDropPanic {
7701        type Output = Result<CheckpointOutput, ExecutionClientError>;
7702
7703        fn poll(
7704            self: std::pin::Pin<&mut Self>,
7705            _cx: &mut std::task::Context<'_>,
7706        ) -> std::task::Poll<Self::Output> {
7707            std::task::Poll::Ready(Ok(CheckpointOutput {
7708                checkpoint_token: Some("token-1".to_owned()),
7709                updated_operations: Vec::new(),
7710                next_marker: None,
7711            }))
7712        }
7713    }
7714
7715    impl Drop for ReadyThenDropPanic {
7716        #[expect(clippy::panic)] // reason: the panic is the behavior under test
7717        fn drop(&mut self) {
7718            panic!("boom in checkpoint drop");
7719        }
7720    }
7721
7722    /// A client whose checkpoint future panics in its destructor.
7723    #[derive(Debug)]
7724    struct DropPanickingClient;
7725
7726    impl ExecutionClient for DropPanickingClient {
7727        fn checkpoint(
7728            &self,
7729            _arn: &str,
7730            _token: &str,
7731            _updates: Vec<OperationUpdate>,
7732        ) -> std::pin::Pin<
7733            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
7734        > {
7735            Box::pin(ReadyThenDropPanic)
7736        }
7737
7738        fn get_state(
7739            &self,
7740            _arn: &str,
7741            _token: &str,
7742            _marker: Option<&str>,
7743        ) -> std::pin::Pin<
7744            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
7745        > {
7746            Box::pin(async {
7747                Ok(GetStateOutput {
7748                    operations: Vec::new(),
7749                    next_marker: None,
7750                })
7751            })
7752        }
7753    }
7754
7755    /// A panic from the destructor of a checkpoint future that returned
7756    /// `Ready` is kept by the coordinator with its original payload, and the
7757    /// caller of the write stays pending.
7758    #[tokio::test]
7759    async fn write_drop_panic_is_recorded_and_caller_stays_pending() {
7760        let payload = write_kept_panic(Arc::new(DropPanickingClient)).await;
7761        assert_eq!(
7762            crate::task::test_support::str_payload(payload.as_ref()),
7763            Some("boom in checkpoint drop")
7764        );
7765    }
7766
7767    #[tokio::test]
7768    async fn child_context_owner_covers_its_descendants() {
7769        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7770        let root = admission_root(&client, Duration::ZERO);
7771        let child = root.new_child("1", None);
7772        root.admission()
7773            .abandon([crate::engine::compute_wire_id_public("1").as_str()]);
7774        let written = tokio::time::timeout(
7775            ADMISSION_BOUND,
7776            child.checkpoint_updates(vec![make_update("u")]),
7777        )
7778        .await;
7779        assert!(written.is_err(), "the child's write never resolves");
7780        tokio::time::timeout(
7781            ADMISSION_BOUND,
7782            root.checkpoint_updates(vec![make_update("p")]),
7783        )
7784        .await
7785        .expect("the root write resolves")
7786        .expect("the root write succeeds");
7787        assert_eq!(recorded_ids(&client), ["p"], "only the root update is sent");
7788    }
7789
7790    #[tokio::test]
7791    async fn finished_child_rejects_escaped_descendant_write() {
7792        use aws_sdk_lambda::types::OperationAction;
7793
7794        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
7795        let root = admission_root(&client, Duration::ZERO);
7796        let slot: Arc<std::sync::Mutex<Option<DurableFuture<u32>>>> =
7797            Arc::new(std::sync::Mutex::new(None));
7798        let release = Arc::new(tokio::sync::Notify::new());
7799        let step_wire = crate::engine::compute_wire_id_public("1-1");
7800        let child_wire = crate::engine::compute_wire_id_public("1");
7801
7802        let returned = tokio::time::timeout(ADMISSION_BOUND, {
7803            let slot = Arc::clone(&slot);
7804            let release = Arc::clone(&release);
7805            let client = Arc::clone(&client);
7806            let step_wire = step_wire.clone();
7807            root.run_in_child_context(move |child| async move {
7808                let handle = child
7809                    .step(move |_| async move {
7810                        release.notified().await;
7811                        Ok(7_u32)
7812                    })
7813                    .name("s")
7814                    .spawn();
7815                *slot.lock().unwrap() = Some(handle);
7816                let mut yields = 0;
7817                while !client
7818                    .recorded_updates()
7819                    .iter()
7820                    .any(|u| u.id == step_wire && u.action == OperationAction::Start)
7821                {
7822                    if yields == 1_000 {
7823                        return Err("the step START was never recorded".into());
7824                    }
7825                    yields += 1;
7826                    tokio::task::yield_now().await;
7827                }
7828                Ok(1_u32)
7829            })
7830            .into_future()
7831        })
7832        .await
7833        .expect("the child returns")
7834        .expect("the child succeeds");
7835        assert_eq!(returned, 1);
7836
7837        release.notify_one();
7838        let handle = slot
7839            .lock()
7840            .unwrap()
7841            .take()
7842            .expect("the body stored the handle");
7843        let awaited = tokio::time::timeout(ADMISSION_BOUND, handle).await;
7844        assert!(awaited.is_err(), "the escaped operation never resolves");
7845
7846        let updates = client.recorded_updates();
7847        let has = |id: &str, action: OperationAction| {
7848            updates.iter().any(|u| u.id == id && u.action == action)
7849        };
7850        assert!(
7851            has(&step_wire, OperationAction::Start),
7852            "s START is recorded"
7853        );
7854        assert!(
7855            has(&child_wire, OperationAction::Succeed),
7856            "the child's SUCCEED is recorded"
7857        );
7858        assert!(
7859            !has(&step_wire, OperationAction::Succeed),
7860            "s SUCCEED is never sent after the child returned"
7861        );
7862    }
7863
7864    // Tenant ID on the root and child spans.
7865
7866    /// Emits one record under the root span of a context built from a
7867    /// Lambda context with `tenant_id`, and one under a child body's span,
7868    /// and returns the span lists of the two records.
7869    fn tenant_span_lists(
7870        tenant_id: Option<&str>,
7871    ) -> (Vec<serde_json::Value>, Vec<serde_json::Value>) {
7872        use crate::tracing_layer::tests::{json_records, json_span_subscriber, spans_of};
7873
7874        let buffer = Arc::new(std::sync::Mutex::new(Vec::new()));
7875        let subscriber = json_span_subscriber(&buffer);
7876        {
7877            let _guard = test_tracing::install_test_subscriber(subscriber);
7878            let mut lambda_context = lambda_runtime::Context::default();
7879            lambda_context.tenant_id = tenant_id.map(str::to_owned);
7880            let root = DurableContext::new_root(
7881                "arn:test".to_owned(),
7882                lambda_context,
7883                Arc::new(CheckpointLog::empty()),
7884            );
7885            root.replay_span().in_scope(|| tracing::info!("in-root"));
7886            let child = root.new_child("1", Some("child"));
7887            child.replay_span().in_scope(|| tracing::info!("in-child"));
7888        };
7889        let records = json_records(&buffer);
7890        (
7891            spans_of(&records, "in-root"),
7892            spans_of(&records, "in-child"),
7893        )
7894    }
7895
7896    #[test]
7897    fn tenant_id_on_root_and_child_spans() {
7898        use crate::tracing_layer::tests::has_field;
7899
7900        let tenant = serde_json::Value::String("t-1".to_owned());
7901        let (root, child) = tenant_span_lists(Some("t-1"));
7902        assert!(
7903            has_field(&root, crate::tracing_layer::fields::TENANT_ID, &tenant),
7904            "{root:?}"
7905        );
7906        assert!(
7907            has_field(&child, crate::tracing_layer::fields::TENANT_ID, &tenant),
7908            "{child:?}"
7909        );
7910    }
7911
7912    #[test]
7913    fn no_tenant_id_without_tenant() {
7914        use crate::tracing_layer::tests::any_field;
7915
7916        let (root, child) = tenant_span_lists(None);
7917        assert!(
7918            !any_field(&root, crate::tracing_layer::fields::TENANT_ID),
7919            "{root:?}"
7920        );
7921        assert!(
7922            !any_field(&child, crate::tracing_layer::fields::TENANT_ID),
7923            "{child:?}"
7924        );
7925        // The record is still under the SDK's spans.
7926        assert!(
7927            any_field(&child, crate::tracing_layer::fields::OPERATION_ID),
7928            "{child:?}"
7929        );
7930    }
7931}
7932
7933#[cfg(test)]
7934// Tests deliberately spawn foreign (unblessed) tasks to exercise runtime
7935// behavior; production spawning is confined to the src/task.rs gateway.
7936#[expect(clippy::disallowed_methods)]
7937mod coordinator_tests {
7938    use super::{DurableContext, remaining_until};
7939    use crate::client::ExecutionClient;
7940    use crate::coordinator::{InvocationCoordinator, PermitKind, Phase};
7941    use crate::driver::test_support::{ScriptedClient, operation_like};
7942    use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
7943    use crate::task::test_support::{Schedule, ScheduleState};
7944    use aws_sdk_lambda::types::{OperationAction, OperationStatus, OperationType, OperationUpdate};
7945    use std::cell::RefCell;
7946    use std::sync::Arc;
7947    use std::sync::atomic::{AtomicBool, Ordering};
7948    use std::time::{Duration, SystemTime};
7949
7950    fn ctx_with(log: Arc<CheckpointLog>, client: &Arc<ScriptedClient>) -> DurableContext {
7951        DurableContext::new_root_with_client(
7952            "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
7953            lambda_runtime::Context::default(),
7954            log,
7955            Arc::clone(client) as Arc<dyn ExecutionClient>,
7956            "t0".to_owned(),
7957        )
7958    }
7959
7960    #[test]
7961    fn remaining_time_cases() {
7962        let now = SystemTime::now();
7963        let now_ms = |offset: i64| {
7964            let base = now
7965                .duration_since(std::time::UNIX_EPOCH)
7966                .expect("after the epoch")
7967                .as_millis();
7968            u64::try_from(i128::try_from(base).expect("fits") + i128::from(offset))
7969                .expect("non-negative")
7970        };
7971        assert_eq!(remaining_until(0, now), None);
7972        let ahead = remaining_until(now_ms(5_000), now).expect("a deadline");
7973        assert!(ahead > Duration::from_secs(4) && ahead <= Duration::from_secs(5));
7974        assert_eq!(remaining_until(now_ms(-1_000), now), Some(Duration::ZERO));
7975
7976        // The context reads its own Lambda deadline.
7977        let ctx = DurableContext::test_context();
7978        assert_eq!(ctx.remaining_time(), None, "the default deadline is 0");
7979    }
7980
7981    /// A wait on a missing record starts its schedule as a `STARTED` record
7982    /// with no end time: a first refresh after 1 s, and a backoff when the
7983    /// refresh finds the record `STARTED`.
7984    #[tokio::test(start_paused = true)]
7985    async fn wait_for_record_without_record_starts_as_started() {
7986        let ctx = DurableContext::test_context();
7987        let coordinator = Arc::clone(ctx.coordinator());
7988        let registered = tokio::time::Instant::now();
7989        let wire = "missing".to_owned();
7990        let mut wait = Box::pin(ctx.wait_for_record(&wire, |s| *s == CheckpointStatus::Succeeded));
7991        let polled = futures_poll_once(wait.as_mut());
7992        assert!(polled.is_pending());
7993        assert_eq!(
7994            coordinator.next_refresh_at(),
7995            Some(registered + Duration::from_secs(1))
7996        );
7997        tokio::time::advance(Duration::from_secs(1)).await;
7998        let crate::coordinator::DueRefresh::Due { keys, .. } =
7999            coordinator.take_due(tokio::time::Instant::now())
8000        else {
8001            unreachable!("the registration is due");
8002        };
8003        let records =
8004            std::collections::BTreeMap::from([(wire.clone(), (CheckpointStatus::Started, None))]);
8005        coordinator.after_refresh(&keys, &records);
8006        assert_eq!(
8007            coordinator.next_refresh_at(),
8008            Some(registered + Duration::from_secs(2)),
8009            "an unchanged STARTED record backs off"
8010        );
8011    }
8012
8013    /// Polls `fut` once with a no-op waker.
8014    fn futures_poll_once<F: Future + ?Sized>(
8015        fut: std::pin::Pin<&mut F>,
8016    ) -> std::task::Poll<F::Output> {
8017        let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
8018        fut.poll(&mut cx)
8019    }
8020
8021    #[test]
8022    fn wait_for_record_ready_before_registration() {
8023        let wire = crate::engine::compute_wire_id_public("1");
8024        let log = Arc::new(CheckpointLog::from_records(vec![(
8025            wire.clone(),
8026            CheckpointRecord::with_status(&wire, CheckpointStatus::Succeeded),
8027        )]));
8028        let client = Arc::new(ScriptedClient::empty());
8029        let ctx = ctx_with(log, &client);
8030        let coordinator = Arc::clone(ctx.coordinator());
8031        let schedule = Schedule::idle(&coordinator);
8032        let mut wait = Box::pin(ctx.wait_for_record(&wire, |s| *s != CheckpointStatus::Started));
8033        assert!(schedule.poll(wait.as_mut()).is_ready());
8034        assert_eq!(coordinator.live_waits(), 0);
8035        assert_eq!(client.call_count(), 0);
8036    }
8037
8038    /// What the readiness hook of the next test changes on its first call:
8039    /// the log, the coordinator, and the wire ID.
8040    type Hook = Option<(Arc<CheckpointLog>, Arc<InvocationCoordinator>, String)>;
8041
8042    thread_local! {
8043        static HOOK: RefCell<Hook> = const { RefCell::new(None) };
8044    }
8045
8046    /// On its first call: marks the record `SUCCEEDED`, wakes its
8047    /// registration, as a merge between the read and the end of the poll
8048    /// does, and returns `false`. Later calls: ready once the record left
8049    /// `STARTED`.
8050    fn merge_during_first_read(status: &CheckpointStatus) -> bool {
8051        if let Some((log, coordinator, wire)) = HOOK.with(|hook| hook.borrow_mut().take()) {
8052            log.insert(
8053                wire.clone(),
8054                CheckpointRecord::with_status(&wire, CheckpointStatus::Succeeded),
8055            );
8056            coordinator.wake_operations(&[wire]);
8057            return false;
8058        }
8059        *status != CheckpointStatus::Started
8060    }
8061
8062    #[test]
8063    fn wait_for_record_merge_after_read_is_not_lost() {
8064        let wire = crate::engine::compute_wire_id_public("1");
8065        let log = Arc::new(CheckpointLog::from_records(vec![(
8066            wire.clone(),
8067            CheckpointRecord::with_status(&wire, CheckpointStatus::Started),
8068        )]));
8069        let client = Arc::new(ScriptedClient::empty());
8070        let ctx = ctx_with(Arc::clone(&log), &client);
8071        let coordinator = Arc::clone(ctx.coordinator());
8072        HOOK.with(|hook| {
8073            *hook.borrow_mut() = Some((Arc::clone(&log), Arc::clone(&coordinator), wire.clone()));
8074        });
8075        let schedule = Schedule::idle(&coordinator);
8076        let mut wait = Box::pin(ctx.wait_for_record(&wire, merge_during_first_read));
8077        assert!(schedule.poll(wait.as_mut()).is_pending());
8078        assert_eq!(
8079            schedule.state(),
8080            ScheduleState::Queued,
8081            "the wake during the poll keeps the task queued"
8082        );
8083        assert_eq!(coordinator.permits(PermitKind::Activity), 1);
8084        assert_eq!(coordinator.phase(), Phase::Running);
8085        assert!(schedule.poll(wait.as_mut()).is_ready());
8086        assert_eq!(
8087            client.call_count(),
8088            0,
8089            "no checkpoint call between the polls"
8090        );
8091        assert_eq!(coordinator.live_waits(), 0);
8092    }
8093
8094    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8095    async fn write_holds_token_lock_through_response_merge() {
8096        let gate = Arc::new(tokio::sync::Notify::new());
8097        let ctx_slot: Arc<std::sync::OnceLock<DurableContext>> =
8098            Arc::new(std::sync::OnceLock::new());
8099        let a_merged_at_b = Arc::new(AtomicBool::new(false));
8100        let responder_ctx = Arc::clone(&ctx_slot);
8101        let responder_flag = Arc::clone(&a_merged_at_b);
8102        let client = Arc::new(
8103            ScriptedClient::new(move |updates, history| {
8104                if history.len() == updates.len() {
8105                    // A's call: the response carries operation `a`.
8106                    return updates
8107                        .iter()
8108                        .map(|u| {
8109                            operation_like(u, OperationStatus::Succeeded)
8110                                .build()
8111                                .expect("all required fields set")
8112                        })
8113                        .collect();
8114                }
8115                let merged = responder_ctx
8116                    .get()
8117                    .is_some_and(|ctx| ctx.has_checkpoint_record("1"));
8118                responder_flag.store(merged, Ordering::SeqCst);
8119                Vec::new()
8120            })
8121            .with_first_call_gate(Arc::clone(&gate)),
8122        );
8123        let ctx = ctx_with(Arc::new(CheckpointLog::empty()), &client);
8124        ctx_slot.set(ctx.clone()).expect("set once");
8125        let update = |positional: &str| {
8126            OperationUpdate::builder()
8127                .id(crate::engine::compute_wire_id_public(positional))
8128                .r#type(OperationType::Step)
8129                .sub_type("Step")
8130                .action(OperationAction::Start)
8131                .build()
8132                .expect("all required fields set")
8133        };
8134        let a_ctx = ctx.clone();
8135        let a_update = update("1");
8136        let a = tokio::spawn(async move { a_ctx.checkpoint_updates_direct(vec![a_update]).await });
8137        for _ in 0..1000 {
8138            if client.call_count() == 1 {
8139                break;
8140            }
8141            tokio::time::sleep(Duration::from_millis(1)).await;
8142        }
8143        assert_eq!(client.call_count(), 1, "A's call was received");
8144        let b_ctx = ctx.clone();
8145        let b_update = update("2");
8146        let b = tokio::spawn(async move { b_ctx.checkpoint_updates_direct(vec![b_update]).await });
8147        for _ in 0..100 {
8148            tokio::task::yield_now().await;
8149        }
8150        assert_eq!(client.call_count(), 1, "B waits for A's merge");
8151        gate.notify_one();
8152        a.await.expect("A does not panic").expect("A succeeds");
8153        b.await.expect("B does not panic").expect("B succeeds");
8154        let calls = client.calls();
8155        assert_eq!(calls.len(), 2);
8156        assert_eq!(
8157            calls.get(1).map(|(token, _)| token.as_str()),
8158            Some("t1"),
8159            "B's call carries the token A's response rotated in"
8160        );
8161        assert!(
8162            a_merged_at_b.load(Ordering::SeqCst),
8163            "operation a was merged before B's call was sent"
8164        );
8165    }
8166
8167    #[tokio::test]
8168    async fn hold_activity_after_suspension_never_resolves() {
8169        let ctx = DurableContext::test_context();
8170        ctx.coordinator().finish();
8171        let ran = Arc::new(AtomicBool::new(false));
8172        let held = Arc::clone(&ran);
8173        let waited = tokio::time::timeout(
8174            Duration::from_millis(100),
8175            ctx.hold_activity(async move { held.store(true, Ordering::SeqCst) }),
8176        )
8177        .await;
8178        assert!(waited.is_err(), "the held future never resolves");
8179        assert!(!ran.load(Ordering::SeqCst), "the work never ran");
8180    }
8181
8182    #[tokio::test]
8183    async fn write_after_suspension_is_rejected() {
8184        let client = Arc::new(ScriptedClient::empty());
8185        let ctx = ctx_with(Arc::new(CheckpointLog::empty()), &client);
8186        ctx.coordinator().finish();
8187        let update = OperationUpdate::builder()
8188            .id("u")
8189            .r#type(OperationType::Step)
8190            .action(OperationAction::Start)
8191            .build()
8192            .expect("all required fields set");
8193        let written = tokio::time::timeout(
8194            Duration::from_millis(100),
8195            ctx.checkpoint_updates_direct(vec![update]),
8196        )
8197        .await;
8198        assert!(written.is_err(), "the rejected write never resolves");
8199        assert_eq!(client.call_count(), 0, "nothing was sent");
8200    }
8201
8202    #[tokio::test]
8203    async fn wait_for_record_after_suspension_never_resolves() {
8204        let wire = crate::engine::compute_wire_id_public("1");
8205        let log = Arc::new(CheckpointLog::from_records(vec![(
8206            wire.clone(),
8207            CheckpointRecord::with_status(&wire, CheckpointStatus::Succeeded),
8208        )]));
8209        let ctx = ctx_with(log, &Arc::new(ScriptedClient::empty()));
8210        ctx.coordinator().finish();
8211        let waited = tokio::time::timeout(
8212            Duration::from_millis(100),
8213            ctx.wait_for_record(&wire, |s| *s != CheckpointStatus::Started),
8214        )
8215        .await;
8216        assert!(waited.is_err(), "a refused registration never resolves");
8217    }
8218
8219    /// A `RETRY` update of `positional` with sub-type `sub_type`.
8220    fn retry_update(positional: &str, sub_type: &str, payload: &str) -> OperationUpdate {
8221        OperationUpdate::builder()
8222            .id(crate::engine::compute_wire_id_public(positional))
8223            .r#type(OperationType::Step)
8224            .sub_type(sub_type)
8225            .action(OperationAction::Retry)
8226            .payload(payload)
8227            .build()
8228            .expect("all required fields set")
8229    }
8230
8231    #[tokio::test]
8232    async fn wfc_retry_keeps_status_the_response_carries() {
8233        let wfc = crate::wait_for_condition::WFC_SUB_TYPE;
8234        for (carried, expected) in [
8235            (OperationStatus::Ready, CheckpointStatus::Ready),
8236            (OperationStatus::Succeeded, CheckpointStatus::Succeeded),
8237            (OperationStatus::Failed, CheckpointStatus::Failed),
8238        ] {
8239            let wire = crate::engine::compute_wire_id_public("1");
8240            let log = Arc::new(CheckpointLog::from_records(vec![(
8241                wire.clone(),
8242                CheckpointRecord::with_status(&wire, CheckpointStatus::Started),
8243            )]));
8244            let status = carried.clone();
8245            let client = Arc::new(ScriptedClient::new(move |updates, _| {
8246                updates
8247                    .iter()
8248                    .map(|u| {
8249                        operation_like(u, status.clone())
8250                            .build()
8251                            .expect("all required fields set")
8252                    })
8253                    .collect()
8254            }));
8255            let ctx = ctx_with(Arc::clone(&log), &client);
8256            ctx.checkpoint_updates_direct(vec![retry_update("1", wfc, "1")])
8257                .await
8258                .expect("the write succeeds");
8259            assert_eq!(
8260                log.get(&wire).expect("present").status,
8261                expected,
8262                "a RETRY response that carries {carried:?} keeps it"
8263            );
8264        }
8265    }
8266
8267    #[tokio::test]
8268    async fn wfc_retry_omitted_from_response_is_pending() {
8269        let wfc = crate::wait_for_condition::WFC_SUB_TYPE;
8270        let ready = crate::engine::compute_wire_id_public("1");
8271        let pending = crate::engine::compute_wire_id_public("2");
8272        let step = crate::engine::compute_wire_id_public("3");
8273        let succeeded = crate::engine::compute_wire_id_public("5");
8274        let mut pending_record = CheckpointRecord::with_status(&pending, CheckpointStatus::Pending);
8275        pending_record.result = Some("old".to_owned());
8276        let log = Arc::new(CheckpointLog::from_records(vec![
8277            (
8278                ready.clone(),
8279                CheckpointRecord::with_status(&ready, CheckpointStatus::Ready),
8280            ),
8281            (pending.clone(), pending_record),
8282            (
8283                step.clone(),
8284                CheckpointRecord::with_status(&step, CheckpointStatus::Started),
8285            ),
8286            (
8287                succeeded.clone(),
8288                CheckpointRecord::with_status(&succeeded, CheckpointStatus::Started),
8289            ),
8290        ]));
8291        let client = Arc::new(ScriptedClient::empty());
8292        let ctx = ctx_with(Arc::clone(&log), &client);
8293        ctx.checkpoint_updates_direct(vec![
8294            retry_update("1", wfc, "next"),
8295            retry_update("2", wfc, "new"),
8296            retry_update("3", "Step", "x"),
8297            retry_update("4", wfc, "x"),
8298            OperationUpdate::builder()
8299                .id(crate::engine::compute_wire_id_public("5"))
8300                .r#type(OperationType::Step)
8301                .sub_type(wfc)
8302                .action(OperationAction::Succeed)
8303                .payload("y")
8304                .build()
8305                .expect("all required fields set"),
8306        ])
8307        .await
8308        .expect("the write succeeds");
8309
8310        let record = log.get(&ready).expect("present");
8311        assert_eq!(
8312            record.status,
8313            CheckpointStatus::Pending,
8314            "a READY record from the previous cycle takes the PENDING the service recorded"
8315        );
8316        assert_eq!(record.result.as_deref(), Some("next"));
8317        assert_eq!(
8318            log.get(&pending).expect("present").result.as_deref(),
8319            Some("old"),
8320            "a record the service already marked PENDING is kept"
8321        );
8322        assert_eq!(
8323            log.get(&step).expect("present").status,
8324            CheckpointStatus::Started,
8325            "a step RETRY is not a wait_for_condition RETRY"
8326        );
8327        assert_eq!(
8328            log.get(&succeeded).expect("present").status,
8329            CheckpointStatus::Started,
8330            "a wait_for_condition SUCCEED is not a RETRY"
8331        );
8332        let missing = crate::engine::compute_wire_id_public("4");
8333        assert!(
8334            log.get(&missing).is_none(),
8335            "a missing record stays missing"
8336        );
8337    }
8338}
8339
8340/// The write order of submissions and of reserved writes.
8341#[cfg(test)]
8342// The tests submit from a task on a runtime worker, which needs a spawn.
8343#[expect(clippy::disallowed_methods)]
8344mod write_order_tests {
8345    use std::sync::Arc;
8346    use std::time::Duration;
8347
8348    use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
8349
8350    use super::DurableContext;
8351    use crate::client::{
8352        CheckpointOutput, ExecutionClient, ExecutionClientError, GetStateOutput,
8353        InMemoryExecutionClient,
8354    };
8355    use crate::engine::CheckpointLog;
8356
8357    fn start(id: &str) -> OperationUpdate {
8358        OperationUpdate::builder()
8359            .id(id.to_owned())
8360            .r#type(OperationType::Step)
8361            .action(OperationAction::Start)
8362            .build()
8363            .expect("all required OperationUpdate fields set")
8364    }
8365
8366    /// A root context over `client`, with the coalescer `window`.
8367    fn context(client: &Arc<InMemoryExecutionClient>, window: Duration) -> DurableContext {
8368        DurableContext::new_root_with_client_and_defaults(
8369            "arn:test".to_owned(),
8370            lambda_runtime::Context::default(),
8371            Arc::new(CheckpointLog::empty()),
8372            Arc::clone(client) as Arc<dyn ExecutionClient>,
8373            "token-0".to_owned(),
8374            window,
8375            0,
8376            None,
8377        )
8378    }
8379
8380    fn ids(client: &InMemoryExecutionClient) -> Vec<String> {
8381        client
8382            .recorded_updates()
8383            .iter()
8384            .map(|u| u.id().to_owned())
8385            .collect()
8386    }
8387
8388    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
8389    async fn submissions_reach_client_in_submission_order() {
8390        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8391        let ctx = context(&client, Duration::ZERO);
8392        tokio::spawn(async move {
8393            let a = ctx.submit_checkpoint_updates(vec![start("A")]);
8394            let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8395            a.await.expect("A is written");
8396            b.await.expect("B is written");
8397        })
8398        .await
8399        .expect("the submitting task completes");
8400        assert_eq!(ids(&client), ["A", "B"]);
8401    }
8402
8403    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8404    async fn child_and_root_submissions_share_write_order() {
8405        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8406        let root = context(&client, Duration::ZERO);
8407        let hold_a = Arc::new(tokio::sync::Notify::new());
8408        let gate_notify = Arc::clone(&hold_a);
8409        root.set_pre_lock_gate(Arc::new(move |updates: &[OperationUpdate]| {
8410            updates
8411                .iter()
8412                .any(|u| u.id() == "A")
8413                .then(|| Arc::clone(&gate_notify))
8414        }));
8415        let child = root.new_child("1", None);
8416        let a = child.submit_checkpoint_updates(vec![start("A")]);
8417        let b = root.submit_checkpoint_updates(vec![start("B")]);
8418        let deadline = tokio::time::Instant::now() + Duration::from_millis(500);
8419        while tokio::time::Instant::now() < deadline {
8420            assert!(
8421                client.recorded_updates().is_empty(),
8422                "a write reached the client while A was held: {:?}",
8423                ids(&client)
8424            );
8425            tokio::time::sleep(Duration::from_millis(10)).await;
8426        }
8427        hold_a.notify_one();
8428        a.await.expect("A is written");
8429        b.await.expect("B is written");
8430        assert_eq!(ids(&client), ["A", "B"]);
8431    }
8432
8433    #[tokio::test]
8434    async fn dropping_reserved_write_releases_in_order() {
8435        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8436        let ctx = context(&client, Duration::ZERO);
8437        let a = ctx.reserve_checkpoint_updates(vec![start("A")]);
8438        let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8439        drop(a);
8440        tokio::time::timeout(Duration::from_secs(2), b)
8441            .await
8442            .map_err(|_| "B is blocked behind the dropped A")
8443            .unwrap()
8444            .expect("B is written");
8445        assert_eq!(ids(&client), ["A", "B"]);
8446    }
8447
8448    /// A released reservation is written in its place: before a write
8449    /// submitted after it, and only once released.
8450    #[tokio::test]
8451    async fn released_reservation_is_written_before_later_submissions() {
8452        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8453        let ctx = context(&client, Duration::ZERO);
8454        let a = ctx.reserve_checkpoint_updates(vec![start("A")]);
8455        let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8456        tokio::time::sleep(Duration::from_millis(50)).await;
8457        assert!(
8458            client.recorded_updates().is_empty(),
8459            "nothing is sent while A is held"
8460        );
8461        let a = a.release();
8462        a.await.expect("A is written");
8463        b.await.expect("B is written");
8464        assert_eq!(ids(&client), ["A", "B"]);
8465    }
8466
8467    /// A fatal record refuses a later update, and does not drop one that
8468    /// was admitted before it: with a zero window and a 60 s window, `A`
8469    /// is written and resolves, and `B` is never written and
8470    /// never resolves.
8471    #[tokio::test]
8472    async fn fatal_record_keeps_admitted_writes() {
8473        for window in [Duration::ZERO, Duration::from_secs(60)] {
8474            let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8475            let ctx = context(&client, window);
8476            let a = ctx.submit_checkpoint_updates(vec![start("A")]);
8477            ctx.coordinator()
8478                .record_fatal("Error".to_owned(), "boom".to_owned());
8479            let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8480            ctx.flush_pending_checkpoints().await;
8481            tokio::time::timeout(Duration::from_secs(2), a)
8482                .await
8483                .map_err(|_| format!("A does not resolve ({window:?})"))
8484                .unwrap()
8485                .expect("A is written");
8486            assert_eq!(ids(&client), ["A"], "{window:?}");
8487            assert!(
8488                tokio::time::timeout(Duration::from_millis(50), b)
8489                    .await
8490                    .is_err(),
8491                "a refused write never resolves ({window:?})"
8492            );
8493            assert_eq!(ids(&client), ["A"], "{window:?}");
8494        }
8495    }
8496
8497    /// A reservation the admission or the coordinator rejects holds
8498    /// nothing, sends nothing, and never resolves.
8499    #[tokio::test]
8500    async fn rejected_reservation_sends_nothing() {
8501        // The phase left `Running`: the coordinator refuses the permit.
8502        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8503        let ctx = context(&client, Duration::ZERO);
8504        ctx.coordinator().finish();
8505        let write = ctx.reserve_checkpoint_updates(vec![start("A")]).release();
8506        assert!(
8507            tokio::time::timeout(Duration::from_millis(50), write)
8508                .await
8509                .is_err(),
8510            "a refused reservation never resolves"
8511        );
8512        assert!(client.recorded_updates().is_empty(), "nothing is sent");
8513        // An abandoned owner: the batch join is rejected.
8514        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8515        let root = context(&client, Duration::ZERO);
8516        let child = root.new_child("1", None);
8517        root.admission()
8518            .abandon([crate::engine::compute_wire_id_public("1").as_str()]);
8519        let write = child.reserve_checkpoint_updates(vec![start("A")]).release();
8520        assert!(
8521            tokio::time::timeout(Duration::from_millis(50), write)
8522                .await
8523                .is_err(),
8524            "a rejected join never resolves"
8525        );
8526        assert_eq!(client.recorded_updates(), Vec::new());
8527    }
8528
8529    /// A client that panics on a request carrying `A`, and records the IDs
8530    /// of the other requests.
8531    #[derive(Default)]
8532    struct PanicsOnA(std::sync::Mutex<Vec<String>>);
8533
8534    impl ExecutionClient for PanicsOnA {
8535        fn checkpoint(
8536            &self,
8537            _execution_arn: &str,
8538            _checkpoint_token: &str,
8539            updates: Vec<OperationUpdate>,
8540        ) -> std::pin::Pin<
8541            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
8542        > {
8543            // The client panic under test.
8544            assert!(!updates.iter().any(|u| u.id() == "A"), "client boom");
8545            self.0
8546                .lock()
8547                .unwrap()
8548                .extend(updates.iter().map(|u| u.id().to_owned()));
8549            Box::pin(async {
8550                Ok(CheckpointOutput::new(
8551                    Some("t".to_owned()),
8552                    Vec::new(),
8553                    None,
8554                ))
8555            })
8556        }
8557
8558        fn get_state(
8559            &self,
8560            _execution_arn: &str,
8561            _checkpoint_token: &str,
8562            _marker: Option<&str>,
8563        ) -> std::pin::Pin<
8564            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
8565        > {
8566            Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
8567        }
8568    }
8569
8570    /// A checkpoint client panic unwinds the batch flusher. The next write
8571    /// starts a new flusher, so it is still sent.
8572    #[tokio::test]
8573    async fn write_after_flusher_panic_is_sent() {
8574        let client = Arc::new(PanicsOnA::default());
8575        let ctx = DurableContext::new_root_with_client(
8576            "arn:test".to_owned(),
8577            lambda_runtime::Context::default(),
8578            Arc::new(CheckpointLog::empty()),
8579            Arc::clone(&client) as Arc<dyn ExecutionClient>,
8580            "token-0".to_owned(),
8581        );
8582        let a = ctx.submit_checkpoint_updates(vec![start("A")]);
8583        assert!(
8584            tokio::time::timeout(Duration::from_millis(50), a)
8585                .await
8586                .is_err(),
8587            "the panicked write never resolves"
8588        );
8589        assert!(
8590            ctx.coordinator().has_panic(),
8591            "the gateway recorded the panic"
8592        );
8593        let b = ctx.submit_checkpoint_updates(vec![start("B")]);
8594        tokio::time::timeout(Duration::from_secs(2), b)
8595            .await
8596            .expect("a new writer sends B")
8597            .expect("B is written");
8598        assert_eq!(*client.0.lock().unwrap(), ["B"]);
8599    }
8600}
8601
8602/// The context usage error for an operation claimed, spawned or executed on
8603/// a task or thread that does not own the context.
8604pub(crate) fn foreign_caller_message(method: &str) -> String {
8605    format!(
8606        "context usage error in \"{method}\": the operation was called on a task or thread \
8607         that does not own the context. Start concurrent operations with .spawn(), map, \
8608         parallel or a combinator, not with tokio::spawn or std::thread::spawn."
8609    )
8610}
8611
8612/// Tests of the change sets the write path queues for
8613/// `on_operation_change`, and of the points that deliver them.
8614#[cfg(test)]
8615mod change_hook_tests {
8616    use super::{CheckpointCoalescer, DurableContext};
8617    use crate::OperationStatus;
8618    use crate::client::{ExecutionClient, InMemoryExecutionClient, TestResponse};
8619    use crate::driver::test_support::{
8620        RefreshClient, ScriptedClient, drive_handler, named, operation_like,
8621    };
8622    use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
8623    use crate::plugin::test_support::Changes;
8624    use crate::plugin::{InvocationPlugins, OperationInfo, PluginFactory};
8625    use aws_sdk_lambda::types::{
8626        Operation, OperationAction, OperationStatus as WireStatus, OperationType, OperationUpdate,
8627    };
8628    use std::sync::{Arc, Mutex, OnceLock};
8629    use std::time::Duration;
8630
8631    /// A root context on `client` over `log`, with `plugins`.
8632    fn ctx_on(
8633        client: Arc<dyn ExecutionClient>,
8634        log: Arc<CheckpointLog>,
8635        window: Duration,
8636        plugins: Arc<InvocationPlugins>,
8637    ) -> DurableContext {
8638        DurableContext::new_root_with_client_and_defaults(
8639            "arn:test".to_owned(),
8640            lambda_runtime::Context::default(),
8641            log,
8642            client,
8643            "t0".to_owned(),
8644            window,
8645            0,
8646            Some(plugins),
8647        )
8648    }
8649
8650    /// A client whose response carries each update's operation, with the
8651    /// status its action records.
8652    fn echo_client() -> Arc<ScriptedClient> {
8653        Arc::new(ScriptedClient::new(|updates, _history| {
8654            updates
8655                .iter()
8656                .map(|u| {
8657                    let status = match u.action() {
8658                        OperationAction::Succeed => WireStatus::Succeeded,
8659                        OperationAction::Fail => WireStatus::Failed,
8660                        OperationAction::Retry => WireStatus::Pending,
8661                        _ => WireStatus::Started,
8662                    };
8663                    operation_like(u, status).build().expect("all fields set")
8664                })
8665                .collect()
8666        }))
8667    }
8668
8669    /// A step operation `id` with `status`.
8670    fn step_op(id: &str, status: WireStatus) -> Operation {
8671        Operation::builder()
8672            .id(id)
8673            .r#type(OperationType::Step)
8674            .status(status)
8675            .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
8676            .build()
8677            .expect("all fields set")
8678    }
8679
8680    /// A step `START` update of `id`.
8681    fn start(id: &str) -> OperationUpdate {
8682        OperationUpdate::builder()
8683            .id(id)
8684            .r#type(OperationType::Step)
8685            .action(OperationAction::Start)
8686            .build()
8687            .expect("all fields set")
8688    }
8689
8690    /// The change hook runs with the token mutex and the coalescer's writer
8691    /// lock free.
8692    #[tokio::test]
8693    async fn change_hook_runs_outside_locks() {
8694        type Locks = (Arc<tokio::sync::Mutex<String>>, Arc<CheckpointCoalescer>);
8695        let locks: Arc<OnceLock<Locks>> = Arc::default();
8696        let free: Arc<Mutex<Vec<(bool, bool)>>> = Arc::default();
8697        let mut recorder = Changes::default();
8698        {
8699            let locks = Arc::clone(&locks);
8700            let free = Arc::clone(&free);
8701            recorder.on_change = Some(Arc::new(move |_info| {
8702                let (token, coalescer) = locks.get().expect("set before the run");
8703                let token_free = token.try_lock().is_ok();
8704                let writer_free = coalescer.writer_lock().try_lock().is_ok();
8705                free.lock().unwrap().push((token_free, writer_free));
8706            }));
8707        }
8708        let ctx = ctx_on(
8709            echo_client(),
8710            Arc::new(CheckpointLog::empty()),
8711            Duration::ZERO,
8712            recorder.plugins(),
8713        );
8714        locks
8715            .set((
8716                ctx.checkpoint_token_mutex(),
8717                Arc::clone(&ctx.inner.coalescer),
8718            ))
8719            .map_err(|_| "set twice")
8720            .expect("set once");
8721        drive_handler(ctx, None, |ctx| async move {
8722            ctx.step(|_| async { Ok(1_u32) })
8723                .name("s")
8724                .await
8725                .map(|v| v.to_string())
8726                .map_err(|e| crate::error::WireError::new(None::<String>, Some(e.to_string())))
8727        })
8728        .await
8729        .expect("no panic");
8730        let free = free.lock().unwrap().clone();
8731        assert!(!free.is_empty(), "at least one call");
8732        assert!(
8733            free.iter().all(|&(token, writer)| token && writer),
8734            "{free:?}"
8735        );
8736    }
8737
8738    /// The retained `START` write of a default step, which its start step
8739    /// reserved, delivers the `STARTED` change when the step awaits it: the
8740    /// change is reported before the step sends its `SUCCEED`.
8741    #[tokio::test]
8742    async fn retained_start_write_reports_its_change() {
8743        let recorder = Changes::default();
8744        let seen: Arc<Mutex<Vec<(OperationAction, usize)>>> = Arc::default();
8745        let client = {
8746            let calls = Arc::clone(&recorder.calls);
8747            let seen = Arc::clone(&seen);
8748            Arc::new(ScriptedClient::new(move |updates, _history| {
8749                let delivered = calls.lock().unwrap().len();
8750                updates
8751                    .iter()
8752                    .map(|u| {
8753                        seen.lock().unwrap().push((u.action().clone(), delivered));
8754                        let status = if u.action() == &OperationAction::Succeed {
8755                            WireStatus::Succeeded
8756                        } else {
8757                            WireStatus::Started
8758                        };
8759                        operation_like(u, status).build().expect("all fields set")
8760                    })
8761                    .collect()
8762            }))
8763        };
8764        let ctx = ctx_on(
8765            client,
8766            Arc::new(CheckpointLog::empty()),
8767            Duration::ZERO,
8768            recorder.plugins(),
8769        );
8770        drive_handler(ctx, None, |ctx| async move {
8771            ctx.step(|_| async { Ok(1_u32) })
8772                .name("s")
8773                .await
8774                .map(|v| v.to_string())
8775                .map_err(|e| crate::error::WireError::new(None::<String>, Some(e.to_string())))
8776        })
8777        .await
8778        .expect("no panic");
8779        let seen = seen.lock().unwrap().clone();
8780        assert_eq!(
8781            seen,
8782            [(OperationAction::Start, 0), (OperationAction::Succeed, 1)],
8783            "{:?}",
8784            recorder.timeline()
8785        );
8786    }
8787
8788    /// A context whose plugin's first change hook, for change set
8789    /// `earlier`, is held open on another thread, and a completed write
8790    /// whose own change set `own` is queued behind it.
8791    struct HeldWrite {
8792        ctx: DurableContext,
8793        recorder: Changes,
8794        write: std::pin::Pin<Box<super::SubmittedWrite>>,
8795        release: std::sync::mpsc::Sender<()>,
8796        holder: Option<std::thread::JoinHandle<()>>,
8797    }
8798
8799    impl HeldWrite {
8800        fn start() -> Self {
8801            Self::start_with(Ok(crate::client::CheckpointOutput::new(
8802                None,
8803                Vec::new(),
8804                None,
8805            )))
8806        }
8807
8808        /// As [`Self::start`], with `result` as the write's result.
8809        fn start_with(
8810            result: Result<crate::client::CheckpointOutput, crate::client::ExecutionClientError>,
8811        ) -> Self {
8812            let (entered_tx, entered_rx) = std::sync::mpsc::channel::<()>();
8813            let (release, release_rx) = std::sync::mpsc::channel::<()>();
8814            let release_rx = Mutex::new(release_rx);
8815            let first = std::sync::atomic::AtomicBool::new(true);
8816            let recorder = Changes {
8817                on_change: Some(Arc::new(move |_info| {
8818                    if first.swap(false, std::sync::atomic::Ordering::SeqCst) {
8819                        entered_tx.send(()).unwrap();
8820                        release_rx.lock().unwrap().recv().unwrap();
8821                    }
8822                })),
8823                ..Changes::default()
8824            };
8825            let ctx = ctx_on(
8826                Arc::new(InMemoryExecutionClient::new(Vec::new())),
8827                Arc::new(CheckpointLog::empty()),
8828                Duration::ZERO,
8829                recorder.plugins(),
8830            );
8831            let entry = |id: &str| {
8832                vec![OperationInfo::from_record(
8833                    id,
8834                    &CheckpointRecord::with_status(id, CheckpointStatus::Started),
8835                )]
8836            };
8837            let plugins = ctx.shared_plugins().expect("the context has plugins");
8838            plugins.push_operation_change(entry("earlier"));
8839            let holder = {
8840                let ctx = ctx.clone();
8841                std::thread::spawn(move || {
8842                    ctx.plugins()
8843                        .expect("the context has plugins")
8844                        .drain_operation_changes(&ctx.inner.engine.checkpoint_log);
8845                })
8846            };
8847            entered_rx.recv().unwrap();
8848            // The write's own response queued its change set.
8849            plugins.push_operation_change(entry("own"));
8850            let (sender, receiver) = tokio::sync::oneshot::channel();
8851            let write = Box::pin(super::SubmittedWrite::from_receiver(receiver).reported_by(&ctx));
8852            assert!(sender.send(result).is_ok(), "the write is waiting");
8853            Self {
8854                ctx,
8855                recorder,
8856                write,
8857                release,
8858                holder: Some(holder),
8859            }
8860        }
8861
8862        /// Waits until the holder delivered both change sets and returned.
8863        fn join_holder(&mut self) {
8864            self.holder
8865                .take()
8866                .expect("the holder is joined once")
8867                .join()
8868                .unwrap();
8869        }
8870
8871        fn waiter_count(&self) -> usize {
8872            self.ctx
8873                .plugins()
8874                .expect("the context has plugins")
8875                .delivery_waiter_count()
8876        }
8877    }
8878
8879    /// A completed write whose change set another thread is still
8880    /// delivering, held open by an earlier change hook, returns its
8881    /// result only after that change set was delivered. So the operation
8882    /// that awaits it fires its next hook after its change. Repeated
8883    /// polls keep one waker registration, and the delivery wakes the
8884    /// registered waker by itself: the test sees the wake before it polls
8885    /// the write again.
8886    #[test]
8887    fn write_waits_for_change_delivered_by_another_task() {
8888        let mut held = HeldWrite::start();
8889        let (signal, signals) = crate::plugin::test_support::SignalWaker::new();
8890        let waker = std::task::Waker::from(Arc::clone(&signal));
8891        let mut cx = std::task::Context::from_waker(&waker);
8892        for _ in 0..100 {
8893            assert!(
8894                held.write.as_mut().poll(&mut cx).is_pending(),
8895                "the write waits while its change set is undelivered"
8896            );
8897        }
8898        assert_eq!(held.waiter_count(), 1);
8899        assert_eq!(held.recorder.updated_ids(), [["earlier"]]);
8900        assert_eq!(signal.woken(), 0);
8901        held.release.send(()).unwrap();
8902        signals
8903            .recv_timeout(Duration::from_secs(30))
8904            .expect("a delivery woke the write's waker");
8905        held.join_holder();
8906        match held.write.as_mut().poll(&mut cx) {
8907            std::task::Poll::Ready(result) => result.expect("the write succeeded"),
8908            std::task::Poll::Pending => unreachable!("the write is ready once delivered"),
8909        };
8910        assert_eq!(held.recorder.updated_ids(), [["earlier"], ["own"]]);
8911        assert_eq!(held.waiter_count(), 0);
8912    }
8913
8914    /// A write dropped while it waits for a delivery removes its waker
8915    /// registration.
8916    #[test]
8917    fn dropped_write_removes_its_registration() {
8918        let mut held = HeldWrite::start();
8919        let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
8920        assert!(held.write.as_mut().poll(&mut cx).is_pending());
8921        assert_eq!(held.waiter_count(), 1);
8922        held.write = Box::pin(super::SubmittedWrite::rejected());
8923        assert_eq!(held.waiter_count(), 0);
8924        held.release.send(()).unwrap();
8925        held.join_holder();
8926        assert_eq!(held.recorder.updated_ids(), [["earlier"], ["own"]]);
8927        assert_eq!(held.waiter_count(), 0);
8928    }
8929
8930    /// The operations maps and change infos one run of a step and a child
8931    /// context builds, with `plugins` or without any.
8932    async fn change_work_of_run(plugins: Option<Arc<InvocationPlugins>>) -> (usize, usize) {
8933        let ctx = DurableContext::new_root_with_client_and_defaults(
8934            "arn:test".to_owned(),
8935            lambda_runtime::Context::default(),
8936            Arc::new(CheckpointLog::empty()),
8937            echo_client(),
8938            "t0".to_owned(),
8939            Duration::ZERO,
8940            0,
8941            plugins,
8942        );
8943        let projections = crate::engine::projections();
8944        let infos = crate::plugin::infos_from_records();
8945        drive_handler(ctx, None, |ctx| async move {
8946            let one = ctx.step(|_| async { Ok(1_u32) }).name("s").await;
8947            let two = ctx
8948                .run_in_child_context(|child| async move {
8949                    Ok(child.step(|_| async { Ok(2_u32) }).name("inner").await?)
8950                })
8951                .name("c")
8952                .await;
8953            Ok(format!("{one:?} {two:?}"))
8954        })
8955        .await
8956        .expect("no panic");
8957        (
8958            crate::engine::projections() - projections,
8959            crate::plugin::infos_from_records() - infos,
8960        )
8961    }
8962
8963    /// Without a plugin, the checkpoint responses of a run build no
8964    /// operations map and no operation info. With one, the same run builds
8965    /// both, so the counters observe the change work.
8966    #[tokio::test]
8967    async fn no_plugin_builds_no_change_info() {
8968        assert_eq!(change_work_of_run(None).await, (0, 0));
8969        let (projected, infos) = change_work_of_run(Some(Changes::default().plugins())).await;
8970        assert!(projected > 0, "a plugin's run projects the log");
8971        assert!(infos > 0, "a plugin's run builds infos");
8972    }
8973
8974    /// A write delivers its change on the caller's task once the
8975    /// detached flush task returned the result. A write whose state read
8976    /// failed never resolves, and records the failure: a drain point
8977    /// delivers its change.
8978    #[tokio::test]
8979    async fn write_reports_change_on_caller() {
8980        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
8981        client.enqueue_checkpoint_response(TestResponse::Success(vec![step_op(
8982            "d",
8983            WireStatus::Started,
8984        )]));
8985        client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
8986            vec![step_op("d", WireStatus::Succeeded)],
8987            "page-2-token".to_owned(),
8988        ));
8989        client.fail_get_state("injected get_state failure");
8990        let recorder = Changes::default();
8991        let ctx = ctx_on(
8992            client,
8993            Arc::new(CheckpointLog::empty()),
8994            Duration::ZERO,
8995            recorder.plugins(),
8996        );
8997        ctx.checkpoint_updates_direct(vec![start("d")])
8998            .await
8999            .expect("the write succeeds");
9000        assert_eq!(recorder.updated_ids(), [["d"]]);
9001        let failed = ctx.checkpoint_updates_direct(vec![start("d")]);
9002        assert!(
9003            tokio::time::timeout(Duration::from_millis(100), failed)
9004                .await
9005                .is_err(),
9006            "the failed write never resolves"
9007        );
9008        assert!(ctx.coordinator().write_failed());
9009        ctx.report_operation_changes().await;
9010        assert_eq!(recorder.updated_ids(), [["d"], ["d"]]);
9011    }
9012
9013    /// A response that changes no status queues nothing, so no call is
9014    /// made.
9015    #[tokio::test]
9016    async fn unchanged_response_queues_nothing() {
9017        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9018        client.enqueue_checkpoint_response(TestResponse::Success(vec![step_op(
9019            "s",
9020            WireStatus::Started,
9021        )]));
9022        let recorder = Changes::default();
9023        let log = Arc::new(CheckpointLog::from_records(vec![(
9024            "s".to_owned(),
9025            CheckpointRecord::with_status("s", CheckpointStatus::Started),
9026        )]));
9027        let ctx = ctx_on(client, log, Duration::ZERO, recorder.plugins());
9028        ctx.checkpoint_updates(vec![start("s")])
9029            .await
9030            .expect("the write succeeds");
9031        assert!(recorder.calls().is_empty(), "{:?}", recorder.updated_ids());
9032    }
9033
9034    /// An `EXECUTION` operation that changes in the response and again in
9035    /// the state pages is listed once, with its last status.
9036    #[tokio::test]
9037    async fn execution_listed_once_per_change_set() {
9038        let execution = |status: WireStatus| {
9039            Operation::builder()
9040                .id("exec-1")
9041                .r#type(OperationType::Execution)
9042                .status(status)
9043                .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
9044                .build()
9045                .expect("all fields set")
9046        };
9047        let client = Arc::new(InMemoryExecutionClient::new(vec![execution(
9048            WireStatus::Succeeded,
9049        )]));
9050        client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
9051            vec![execution(WireStatus::Started)],
9052            "page-2-token".to_owned(),
9053        ));
9054        let recorder = Changes::default();
9055        let ctx = ctx_on(
9056            client,
9057            Arc::new(CheckpointLog::empty()),
9058            Duration::ZERO,
9059            recorder.plugins(),
9060        );
9061        ctx.checkpoint_updates(Vec::new())
9062            .await
9063            .expect("the write succeeds");
9064        let calls = recorder.calls();
9065        assert_eq!(calls.len(), 1, "{:?}", recorder.updated_ids());
9066        let info = calls.first().expect("one call");
9067        let statuses: Vec<_> = info
9068            .updated_operations()
9069            .iter()
9070            .map(OperationInfo::status)
9071            .collect();
9072        assert_eq!(statuses, [Some(OperationStatus::Succeeded)]);
9073    }
9074
9075    /// When reading the state pages fails, the failure is recorded, and
9076    /// the change the response merged is still reported.
9077    #[tokio::test]
9078    async fn change_reported_when_pagination_hydration_fails() {
9079        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9080        client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
9081            vec![step_op("op-hydration", WireStatus::Started)],
9082            "page-2-token".to_owned(),
9083        ));
9084        client.fail_get_state("injected get_state failure");
9085        let recorder = Changes::default();
9086        let ctx = ctx_on(
9087            client,
9088            Arc::new(CheckpointLog::empty()),
9089            Duration::ZERO,
9090            recorder.plugins(),
9091        );
9092        let write = ctx.checkpoint_updates(vec![start("op-hydration")]);
9093        assert!(
9094            tokio::time::timeout(Duration::from_millis(100), write)
9095                .await
9096                .is_err(),
9097            "the failed write never resolves"
9098        );
9099        assert!(ctx.coordinator().write_failed());
9100        ctx.report_operation_changes().await;
9101        assert_eq!(recorder.updated_ids(), [["op-hydration"]]);
9102    }
9103
9104    /// The change that completes a waiting operation, merged from a
9105    /// refresh, is reported before the operation's end hook. Without the
9106    /// drain when the wait returns, the sibling step's write at 3 s would
9107    /// deliver it, after the end hook.
9108    #[tokio::test(start_paused = true)]
9109    async fn refresh_reports_wait_change() {
9110        let client = Arc::new(RefreshClient::new(|req| {
9111            if req.empty_number == 1
9112                && let Some(start) = named(req.history, "w", &OperationAction::Start)
9113            {
9114                let record = operation_like(start, WireStatus::Succeeded)
9115                    .build()
9116                    .expect("all required fields set");
9117                return Ok(vec![record]);
9118            }
9119            Ok(Vec::new())
9120        }));
9121        let recorder = Changes::default();
9122        let ctx = ctx_on(
9123            Arc::clone(&client) as Arc<dyn ExecutionClient>,
9124            Arc::new(CheckpointLog::empty()),
9125            Duration::ZERO,
9126            recorder.plugins(),
9127        );
9128        let outcome = drive_handler(ctx, None, |ctx| async move {
9129            let wait = ctx.wait(Duration::from_secs(1)).name("w").spawn();
9130            let step = ctx
9131                .step(|_| async {
9132                    tokio::time::sleep(Duration::from_secs(3)).await;
9133                    Ok(1_u32)
9134                })
9135                .name("s")
9136                .spawn();
9137            let (wait, _step) = tokio::join!(wait, step);
9138            Ok(format!("{}", wait.is_ok()))
9139        })
9140        .await
9141        .expect("no panic");
9142        assert_eq!(
9143            outcome,
9144            crate::driver::InvocationOutcome::Complete("true".to_owned())
9145        );
9146        let succeeded = recorder.calls().iter().any(|info| {
9147            info.updated_operations()
9148                .iter()
9149                .any(|op| op.name() == Some("w") && op.status() == Some(OperationStatus::Succeeded))
9150        });
9151        assert!(succeeded, "{:?}", recorder.timeline());
9152        let timeline = recorder.timeline();
9153        let change = timeline
9154            .iter()
9155            .position(|e| e == "change:w:SUCCEEDED")
9156            .expect("the change is recorded");
9157        let end = timeline
9158            .iter()
9159            .position(|e| e == "end:w")
9160            .expect("the end hook ran");
9161        assert!(change < end, "{timeline:?}");
9162    }
9163
9164    /// The task that sends a refresh delivers the change the refresh
9165    /// merged when the call returns, also for an operation nobody waits
9166    /// on. Without that drain, the change waits for the next drain point:
9167    /// the wait's completion at the second refresh, or the sibling step's
9168    /// write at 3 s.
9169    #[tokio::test(start_paused = true)]
9170    async fn refresh_sender_reports_change() {
9171        let client = Arc::new(RefreshClient::new(|req| {
9172            let Some(start) = named(req.history, "w", &OperationAction::Start) else {
9173                return Ok(Vec::new());
9174            };
9175            let record = match req.empty_number {
9176                1 => step_op("x", WireStatus::Succeeded),
9177                2 => operation_like(start, WireStatus::Succeeded)
9178                    .build()
9179                    .expect("all required fields set"),
9180                _ => return Ok(Vec::new()),
9181            };
9182            Ok(vec![record])
9183        }));
9184        let at: Arc<Mutex<Vec<Duration>>> = Arc::default();
9185        let mut recorder = Changes::default();
9186        {
9187            let at = Arc::clone(&at);
9188            let started = tokio::time::Instant::now();
9189            recorder.on_change = Some(Arc::new(move |info| {
9190                if info.updated_operations().iter().any(|op| op.id() == "x") {
9191                    at.lock().unwrap().push(started.elapsed());
9192                }
9193            }));
9194        }
9195        let ctx = ctx_on(
9196            Arc::clone(&client) as Arc<dyn ExecutionClient>,
9197            Arc::new(CheckpointLog::empty()),
9198            Duration::ZERO,
9199            recorder.plugins(),
9200        );
9201        drive_handler(ctx, None, |ctx| async move {
9202            let wait = ctx.wait(Duration::from_secs(1)).name("w").spawn();
9203            let step = ctx
9204                .step(|_| async {
9205                    tokio::time::sleep(Duration::from_secs(3)).await;
9206                    Ok(1_u32)
9207                })
9208                .name("s")
9209                .spawn();
9210            let (wait, _step) = tokio::join!(wait, step);
9211            Ok(format!("{}", wait.is_ok()))
9212        })
9213        .await
9214        .expect("no panic");
9215        let at = at.lock().unwrap().clone();
9216        assert_eq!(at.len(), 1, "{:?}", recorder.timeline());
9217        assert!(
9218            at.first()
9219                .is_some_and(|t| *t < Duration::from_millis(1_500)),
9220            "the change of x is reported at the first refresh: {at:?}, {:?}",
9221            client.empty_answers()
9222        );
9223    }
9224
9225    /// Runs 50 new steps over a log of 2 000 succeeded steps with the
9226    /// given number of recording plugins, and returns the projections the
9227    /// run made and the number of change calls the first plugin received.
9228    async fn projections_of_run(plugin_count: usize) -> (usize, usize) {
9229        let records = (0..2_000)
9230            .map(|i| {
9231                let id = format!("old-{i}");
9232                let mut record = CheckpointRecord::with_status(&id, CheckpointStatus::Succeeded);
9233                record.op_type = Some("STEP".to_owned());
9234                (id, record)
9235            })
9236            .collect();
9237        let recorders: Vec<Changes> = (0..plugin_count).map(|_| Changes::default()).collect();
9238        let plugins = crate::plugin::test_support::plugins_of_all(
9239            recorders
9240                .iter()
9241                .map(|r| Arc::new(r.clone()) as Arc<dyn PluginFactory>)
9242                .collect(),
9243        );
9244        let ctx = ctx_on(
9245            echo_client(),
9246            Arc::new(CheckpointLog::from_records(records)),
9247            Duration::ZERO,
9248            plugins,
9249        );
9250        let before = crate::engine::projections();
9251        drive_handler(ctx, None, |ctx| async move {
9252            for i in 0..50_u32 {
9253                ctx.step(move |_| async move { Ok(i) })
9254                    .name(format!("new-{i}"))
9255                    .await
9256                    .map_err(|e| {
9257                        crate::error::WireError::new(None::<String>, Some(e.to_string()))
9258                    })?;
9259            }
9260            Ok("done".to_owned())
9261        })
9262        .await
9263        .expect("no panic");
9264        let projected = crate::engine::projections() - before;
9265        let calls = recorders.first().expect("a plugin").calls().len();
9266        (projected, calls)
9267    }
9268
9269    /// Each change event projects the log once, whatever the number of
9270    /// plugins.
9271    #[tokio::test]
9272    async fn change_hooks_project_once_per_response() {
9273        let (one_projected, one_calls) = projections_of_run(1).await;
9274        let (three_projected, three_calls) = projections_of_run(3).await;
9275        assert!(one_calls > 0);
9276        assert_eq!(one_projected, one_calls);
9277        assert_eq!(three_projected, three_calls);
9278        assert_eq!(one_calls, three_calls);
9279    }
9280
9281    /// An `EXECUTION` operation in a response replaces the plugins' slot,
9282    /// is listed as a change, and does not enter the log.
9283    #[tokio::test]
9284    async fn execution_update_replaces_slot() {
9285        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9286        let execution = Operation::builder()
9287            .id("exec-1")
9288            .r#type(OperationType::Execution)
9289            .status(WireStatus::Succeeded)
9290            .start_timestamp(aws_smithy_types::DateTime::from_secs(0))
9291            .build()
9292            .expect("all fields set");
9293        client.enqueue_checkpoint_response(TestResponse::Success(vec![
9294            execution,
9295            step_op("s", WireStatus::Started),
9296        ]));
9297        let recorder = Changes::default();
9298        let factories: Vec<(&'static str, Arc<dyn PluginFactory>)> =
9299            vec![("test::Changes", Arc::new(recorder.clone()))];
9300        let slot = OperationInfo::execution(
9301            "exec-1",
9302            "EXECUTION",
9303            &CheckpointStatus::Started,
9304            None,
9305            None,
9306            None,
9307        );
9308        let plugins =
9309            crate::plugin::create_plugins(&factories, slot, crate::plugin::tests_support_info)
9310                .expect("one factory");
9311        let log = Arc::new(CheckpointLog::empty());
9312        let ctx = ctx_on(client, Arc::clone(&log), Duration::ZERO, plugins);
9313        ctx.checkpoint_updates(vec![start("s")])
9314            .await
9315            .expect("the write succeeds");
9316        let calls = recorder.calls();
9317        let info = calls.first().expect("one call");
9318        assert_eq!(
9319            info.operations()
9320                .get("exec-1")
9321                .and_then(OperationInfo::status),
9322            Some(OperationStatus::Succeeded)
9323        );
9324        assert!(
9325            info.updated_operations()
9326                .iter()
9327                .any(|op| op.id() == "exec-1")
9328        );
9329        assert!(!log.contains("exec-1"));
9330    }
9331
9332    /// A paginated response and its state pages give one change set, which
9333    /// lists each changed ID once, with the status after both merges.
9334    #[tokio::test]
9335    async fn paginated_response_is_one_change_set() {
9336        let client = Arc::new(InMemoryExecutionClient::new(vec![
9337            step_op("step-1", WireStatus::Succeeded),
9338            step_op("step-2", WireStatus::Succeeded),
9339        ]));
9340        client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
9341            vec![step_op("step-1", WireStatus::Started)],
9342            "page-2-token".to_owned(),
9343        ));
9344        let recorder = Changes::default();
9345        let ctx = ctx_on(
9346            client,
9347            Arc::new(CheckpointLog::empty()),
9348            Duration::ZERO,
9349            recorder.plugins(),
9350        );
9351        ctx.checkpoint_updates(Vec::new())
9352            .await
9353            .expect("the write succeeds");
9354        assert_eq!(recorder.updated_ids(), [["step-1", "step-2"]]);
9355        let calls = recorder.calls();
9356        let step_1 = calls
9357            .first()
9358            .and_then(|info| info.updated_operations().first())
9359            .expect("step-1 is listed");
9360        assert_eq!(step_1.status(), Some(OperationStatus::Succeeded));
9361    }
9362
9363    /// A waiting operation reads a status that a checkpoint response
9364    /// changed only after the response's change set is queued, so the
9365    /// change is delivered before the operation goes on, also when the
9366    /// operation polls on another thread while the response is merged. The
9367    /// `before_push` hook, which runs after the merge and before the queue,
9368    /// asks the waiter's thread to poll once and waits up to 200 ms for that
9369    /// poll. For an unpaginated response, a paginated one, and a paginated
9370    /// one whose page read fails.
9371    #[tokio::test]
9372    async fn merged_status_is_hidden_until_its_change_is_queued() {
9373        for case in ["unpaginated", "paginated", "failed page read"] {
9374            let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
9375            let response = vec![step_op("d", WireStatus::Succeeded)];
9376            client.enqueue_checkpoint_response(if case == "unpaginated" {
9377                TestResponse::Success(response)
9378            } else {
9379                TestResponse::SuccessPaginated(response, "page-2-token".to_owned())
9380            });
9381            if case == "failed page read" {
9382                client.fail_get_state("injected get_state failure");
9383            }
9384            let recorder = Changes::default();
9385            let log = Arc::new(CheckpointLog::from_records(vec![(
9386                "d".to_owned(),
9387                CheckpointRecord::with_status("d", CheckpointStatus::Started),
9388            )]));
9389            let ctx = ctx_on(client, log, Duration::ZERO, recorder.plugins());
9390            // The waiter's thread polls the wait once per request and replies
9391            // after each poll. When the wait resolves, it records how many
9392            // change sets were delivered by then.
9393            let (poll_tx, poll_rx) = std::sync::mpsc::channel::<()>();
9394            let (reply_tx, reply_rx) = std::sync::mpsc::channel::<()>();
9395            let resolved: Arc<Mutex<Option<usize>>> = Arc::default();
9396            let waiter = {
9397                let ctx = ctx.clone();
9398                let calls = Arc::clone(&recorder.calls);
9399                let resolved = Arc::clone(&resolved);
9400                std::thread::spawn(move || {
9401                    let mut wait =
9402                        Box::pin(ctx.wait_for_record("d", |s| *s == CheckpointStatus::Succeeded));
9403                    let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
9404                    while poll_rx.recv().is_ok() {
9405                        let ready = wait.as_mut().poll(&mut cx).is_ready();
9406                        if ready {
9407                            *resolved.lock().unwrap() = Some(calls.lock().unwrap().len());
9408                        }
9409                        reply_tx.send(()).unwrap();
9410                        if ready {
9411                            return;
9412                        }
9413                    }
9414                })
9415            };
9416            poll_tx.send(()).unwrap();
9417            reply_rx.recv().unwrap();
9418            assert_eq!(
9419                *resolved.lock().unwrap(),
9420                None,
9421                "{case}: the wait registers"
9422            );
9423            let reply_rx = Arc::new(Mutex::new(reply_rx));
9424            {
9425                let poll_tx = poll_tx.clone();
9426                let reply_rx = Arc::clone(&reply_rx);
9427                ctx.shared_plugins()
9428                    .expect("the context has plugins")
9429                    .set_before_push(move |_| {
9430                        poll_tx.send(()).unwrap();
9431                        // A waiter that reads the merged status replies at once.
9432                        let _ = reply_rx
9433                            .lock()
9434                            .unwrap()
9435                            .recv_timeout(Duration::from_millis(200));
9436                    });
9437            }
9438            // A failed page read leaves the write unresolved.
9439            let write = ctx.checkpoint_updates_direct(vec![start("d")]);
9440            let _ = tokio::time::timeout(Duration::from_millis(500), write).await;
9441            for _ in 0..50 {
9442                if resolved.lock().unwrap().is_some() || poll_tx.send(()).is_err() {
9443                    break;
9444                }
9445                let _ = reply_rx
9446                    .lock()
9447                    .unwrap()
9448                    .recv_timeout(Duration::from_millis(100));
9449            }
9450            drop(poll_tx);
9451            waiter.join().unwrap();
9452            assert_eq!(
9453                *resolved.lock().unwrap(),
9454                Some(1),
9455                "{case}: the change is delivered before the wait resolves"
9456            );
9457        }
9458    }
9459}
9460
9461/// Tests of the write paths after a checkpoint response without a token,
9462/// and after a failed checkpoint write.
9463#[cfg(test)]
9464mod withdrawal_write_tests {
9465    use super::{CheckpointCoalescer, DurableContext, WriteError};
9466    use crate::client::{
9467        CheckpointOutput, ExecutionClient, ExecutionClientError, ExecutionClientErrorScope,
9468        GetStateOutput,
9469    };
9470    use crate::coordinator::PermitKind;
9471    use crate::driver::{FatalError, FatalSeverity};
9472    use crate::engine::CheckpointLog;
9473    use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
9474    use std::sync::Arc;
9475    use std::sync::atomic::{AtomicUsize, Ordering};
9476    use std::time::Duration;
9477
9478    /// Answers every checkpoint request without a token, and counts them.
9479    #[derive(Default)]
9480    struct Tokenless(AtomicUsize);
9481
9482    impl ExecutionClient for Tokenless {
9483        fn checkpoint(
9484            &self,
9485            _execution_arn: &str,
9486            _checkpoint_token: &str,
9487            _updates: Vec<OperationUpdate>,
9488        ) -> std::pin::Pin<
9489            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
9490        > {
9491            self.0.fetch_add(1, Ordering::SeqCst);
9492            Box::pin(async { Ok(CheckpointOutput::new(None, Vec::new(), None)) })
9493        }
9494
9495        fn get_state(
9496            &self,
9497            _execution_arn: &str,
9498            _checkpoint_token: &str,
9499            _marker: Option<&str>,
9500        ) -> std::pin::Pin<
9501            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
9502        > {
9503            Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
9504        }
9505    }
9506
9507    fn update(id: &str, action: OperationAction) -> OperationUpdate {
9508        OperationUpdate::builder()
9509            .id(id)
9510            .r#type(OperationType::Step)
9511            .action(action)
9512            .build()
9513            .unwrap()
9514    }
9515
9516    /// A root context over `client`.
9517    fn ctx_with(client: Arc<dyn ExecutionClient>) -> DurableContext {
9518        DurableContext::new_root_with_client(
9519            "arn:test".to_owned(),
9520            lambda_runtime::Context::default(),
9521            Arc::new(CheckpointLog::empty()),
9522            client,
9523            "t0".to_owned(),
9524        )
9525    }
9526
9527    fn ctx_on(client: &Arc<Tokenless>, coalescer: Option<CheckpointCoalescer>) -> DurableContext {
9528        let client = Arc::clone(client) as Arc<dyn ExecutionClient>;
9529        let log = Arc::new(CheckpointLog::empty());
9530        match coalescer {
9531            Some(coalescer) => DurableContext::new_root_with_client_and_coalescer(
9532                "arn:test".to_owned(),
9533                lambda_runtime::Context::default(),
9534                log,
9535                client,
9536                "t0".to_owned(),
9537                coalescer,
9538            ),
9539            None => DurableContext::new_root_with_client(
9540                "arn:test".to_owned(),
9541                lambda_runtime::Context::default(),
9542                log,
9543                client,
9544                "t0".to_owned(),
9545            ),
9546        }
9547    }
9548
9549    /// After the latch, a request sends nothing: a write after the drive
9550    /// returns `Withdrawn` without calling the transport.
9551    #[tokio::test]
9552    async fn write_after_withdrawal_sends_nothing() {
9553        let client = Arc::new(Tokenless::default());
9554        let ctx = ctx_on(&client, None);
9555        let first = ctx.checkpoint_updates_after_drive(vec![update("a", OperationAction::Start)]);
9556        assert!(matches!(first.await, Err(WriteError::Stopped)));
9557        assert!(ctx.coordinator().withdrawn());
9558        assert_eq!(client.0.load(Ordering::SeqCst), 1);
9559        let second = ctx.checkpoint_updates_after_drive(vec![update("b", OperationAction::Start)]);
9560        assert!(matches!(second.await, Err(WriteError::Stopped)));
9561        assert_eq!(
9562            client.0.load(Ordering::SeqCst),
9563            1,
9564            "the second request is not sent"
9565        );
9566    }
9567
9568    /// Fails request 1 with an execution-scoped error, answers every later
9569    /// request with a token, and counts the requests.
9570    #[derive(Default)]
9571    struct FailFirst(AtomicUsize);
9572
9573    impl ExecutionClient for FailFirst {
9574        fn checkpoint(
9575            &self,
9576            _execution_arn: &str,
9577            _checkpoint_token: &str,
9578            _updates: Vec<OperationUpdate>,
9579        ) -> std::pin::Pin<
9580            Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
9581        > {
9582            let call = self.0.fetch_add(1, Ordering::SeqCst);
9583            Box::pin(async move {
9584                if call == 0 {
9585                    Err(ExecutionClientError::new(
9586                        ExecutionClientErrorScope::Execution,
9587                        "down",
9588                    ))
9589                } else {
9590                    Ok(CheckpointOutput::new(
9591                        Some(format!("t{call}")),
9592                        Vec::new(),
9593                        None,
9594                    ))
9595                }
9596            })
9597        }
9598
9599        fn get_state(
9600            &self,
9601            _execution_arn: &str,
9602            _checkpoint_token: &str,
9603            _marker: Option<&str>,
9604        ) -> std::pin::Pin<
9605            Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
9606        > {
9607            Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
9608        }
9609    }
9610
9611    /// Yields until `done` holds, so the batch flusher runs.
9612    async fn until(done: impl Fn() -> bool) {
9613        for _ in 0..1000 {
9614            if done() {
9615                return;
9616            }
9617            tokio::task::yield_now().await;
9618        }
9619        assert!(done(), "the condition never held");
9620    }
9621
9622    /// A context over [`FailFirst`] whose first write, `a`'s START,
9623    /// failed. Returns the failed write's unresolved future.
9624    async fn failed_first_write() -> (DurableContext, Arc<FailFirst>, super::SubmittedWrite) {
9625        let client = Arc::new(FailFirst::default());
9626        let ctx = ctx_with(Arc::clone(&client) as Arc<dyn ExecutionClient>);
9627        let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9628        until(|| ctx.coordinator().write_failed()).await;
9629        (ctx, client, write)
9630    }
9631
9632    /// A failed write sets the latch and records its classification for
9633    /// its first update, and its caller never resolves.
9634    #[tokio::test(start_paused = true)]
9635    async fn failed_write_records_and_never_resolves() {
9636        let (ctx, client, write) = failed_first_write().await;
9637        let fatal = ctx.coordinator().fatal_error().expect("recorded");
9638        assert_eq!(
9639            fatal.error_type,
9640            crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
9641        );
9642        assert_eq!(
9643            ctx.coordinator().write_failure().map(|f| f.error_message),
9644            Some(fatal.error_message)
9645        );
9646        assert!(
9647            tokio::time::timeout(Duration::from_secs(60), write)
9648                .await
9649                .is_err(),
9650            "the failed write never resolves"
9651        );
9652        assert_eq!(client.0.load(Ordering::SeqCst), 1);
9653    }
9654
9655    /// A reserved write released after a failed write sends nothing and
9656    /// never resolves.
9657    #[tokio::test(start_paused = true)]
9658    async fn reserved_write_after_failure_sends_nothing() {
9659        let (ctx, client, _failed) = failed_first_write().await;
9660        let reserved = ctx.reserve_checkpoint_updates(vec![update("b", OperationAction::Start)]);
9661        let write = reserved.release();
9662        assert!(
9663            tokio::time::timeout(Duration::from_secs(60), write)
9664                .await
9665                .is_err(),
9666            "the reserved write never resolves"
9667        );
9668        assert_eq!(
9669            client.0.load(Ordering::SeqCst),
9670            1,
9671            "the reserved write is not sent"
9672        );
9673    }
9674
9675    /// A write after the drive, after a write failed during the drive,
9676    /// sends nothing and returns at once.
9677    #[tokio::test(start_paused = true)]
9678    async fn write_after_drive_after_failure_sends_nothing() {
9679        let (ctx, client, _failed) = failed_first_write().await;
9680        ctx.coordinator().finish();
9681        let written = ctx
9682            .checkpoint_updates_after_drive(vec![update("b", OperationAction::Start)])
9683            .await;
9684        assert!(matches!(written, Err(WriteError::Stopped)), "{written:?}");
9685        assert_eq!(client.0.load(Ordering::SeqCst), 1, "nothing is sent");
9686    }
9687
9688    /// A write without an execution client records the failure for its
9689    /// first update, and never resolves.
9690    #[tokio::test(start_paused = true)]
9691    async fn write_without_client_records_the_failure() {
9692        let ctx = DurableContext::test_context();
9693        let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9694        assert!(
9695            tokio::time::timeout(Duration::from_secs(60), write)
9696                .await
9697                .is_err(),
9698            "the write never resolves"
9699        );
9700        let fatal = ctx.coordinator().write_failure().expect("latched");
9701        assert_eq!(
9702            fatal.error_type,
9703            crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE
9704        );
9705        assert!(
9706            fatal
9707                .error_message
9708                .contains("no execution client configured"),
9709            "{fatal:?}"
9710        );
9711    }
9712
9713    /// A write without an execution client records its failure under the
9714    /// checkpoint-token lock: while another write holds the lock, nothing
9715    /// is recorded.
9716    #[tokio::test(start_paused = true)]
9717    async fn write_without_client_records_under_the_token_lock() {
9718        let ctx = DurableContext::test_context();
9719        let held = ctx.inner.checkpoint_token.lock().await;
9720        let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9721        tokio::time::sleep(Duration::from_secs(60)).await;
9722        assert!(
9723            !ctx.coordinator().write_failed(),
9724            "nothing is recorded while the lock is held"
9725        );
9726        assert!(ctx.coordinator().fatal_error().is_none());
9727        drop(held);
9728        until(|| ctx.coordinator().write_failed()).await;
9729        drop(write);
9730    }
9731
9732    /// A write without an execution client that reaches the lock after
9733    /// the checkpoint token was withdrawn records nothing: the latch is
9734    /// checked before the client.
9735    #[tokio::test(start_paused = true)]
9736    async fn write_without_client_after_withdrawal_records_nothing() {
9737        let ctx = DurableContext::test_context();
9738        ctx.coordinator().withdraw("arn:test", "req", || {});
9739        let written = ctx
9740            .checkpoint_direct_with_events(vec![update("a", OperationAction::Start)], &[], None)
9741            .await;
9742        assert!(matches!(written, Err(WriteError::Stopped)), "{written:?}");
9743        assert!(!ctx.coordinator().write_failed());
9744        assert!(ctx.coordinator().fatal_error().is_none());
9745    }
9746
9747    /// A failed write drops the coalescer's open batch, and a later
9748    /// join is rejected: no contributor resolves, and the end-of-invocation
9749    /// flush sends nothing.
9750    #[tokio::test(start_paused = true)]
9751    async fn failed_write_drops_the_open_batch_and_rejects_later_joins() {
9752        let client = Arc::new(FailFirst::default());
9753        let ctx = DurableContext::new_root_with_client_and_coalescer(
9754            "arn:test".to_owned(),
9755            lambda_runtime::Context::default(),
9756            Arc::new(CheckpointLog::empty()),
9757            Arc::clone(&client) as Arc<dyn ExecutionClient>,
9758            "t0".to_owned(),
9759            CheckpointCoalescer::new(Duration::from_secs(60)),
9760        );
9761        let buffered = ctx.submit_checkpoint_updates(vec![update("b", OperationAction::Start)]);
9762        let failed = ctx
9763            .checkpoint_updates_after_drive(vec![update("a", OperationAction::Start)])
9764            .await;
9765        assert!(matches!(failed, Err(WriteError::Failed(_))), "{failed:?}");
9766        assert!(ctx.coordinator().write_failed());
9767        assert!(
9768            ctx.queued_update_ids().is_empty(),
9769            "the open batch is dropped"
9770        );
9771        let later = ctx.submit_checkpoint_updates(vec![update("c", OperationAction::Start)]);
9772        assert!(
9773            ctx.queued_update_ids().is_empty(),
9774            "the later join is rejected"
9775        );
9776        ctx.flush_pending_checkpoints().await;
9777        let all = async { tokio::join!(buffered, later) };
9778        assert!(
9779            tokio::time::timeout(Duration::from_secs(120), all)
9780                .await
9781                .is_err(),
9782            "no contributor resolves"
9783        );
9784        assert_eq!(client.0.load(Ordering::SeqCst), 1, "nothing more is sent");
9785    }
9786
9787    /// A guard the race tests hold; any value is one.
9788    trait Held {}
9789    impl<T> Held for T {}
9790
9791    /// Submits `b`'s START on one thread while the test holds the buffer
9792    /// lock `lock_queue` takes. The submitting thread signals when it
9793    /// reaches `point`, just before it takes that lock, so it has passed
9794    /// every check made before the lock. A write failure then sets the
9795    /// latch, and the lock is released. The submission must see the latch
9796    /// under the lock: nothing waits to be sent, and nothing is sent. A
9797    /// failed write drops a submission queued before the latch (see
9798    /// `failed_write_drops_the_open_batch_and_rejects_later_joins`).
9799    fn submission_racing_a_failed_write(
9800        coalescer: CheckpointCoalescer,
9801        point: crate::coordinator::race_points::RacePoint,
9802        lock_queue: fn(&DurableContext) -> Box<dyn Held + '_>,
9803    ) {
9804        let runtime = tokio::runtime::Builder::new_current_thread()
9805            .enable_all()
9806            .build()
9807            .unwrap();
9808        let client = Arc::new(FailFirst::default());
9809        let ctx = {
9810            let _entered = runtime.enter();
9811            let client = Arc::clone(&client) as Arc<dyn ExecutionClient>;
9812            DurableContext::new_root_with_client_and_coalescer(
9813                "arn:test".to_owned(),
9814                lambda_runtime::Context::default(),
9815                Arc::new(CheckpointLog::empty()),
9816                client,
9817                "t0".to_owned(),
9818                coalescer,
9819            )
9820        };
9821        let held = lock_queue(&ctx);
9822        let (reached_tx, reached_rx) = std::sync::mpsc::channel();
9823        let submitter = {
9824            let ctx = ctx.clone();
9825            let handle = runtime.handle().clone();
9826            std::thread::spawn(move || {
9827                let _entered = handle.enter();
9828                crate::coordinator::race_points::set(move |reached| {
9829                    if reached == point {
9830                        let _ = reached_tx.send(());
9831                    }
9832                });
9833                drop(ctx.submit_checkpoint_updates(vec![update("b", OperationAction::Start)]));
9834            })
9835        };
9836        // The submitter is past every check made before the lock, and the
9837        // latch is not set yet.
9838        reached_rx.recv().unwrap();
9839        ctx.coordinator().fail_write(
9840            FatalError {
9841                error_type: "Failed".to_owned(),
9842                error_message: "failed".to_owned(),
9843                severity: FatalSeverity::FailExecution,
9844            },
9845            || {},
9846        );
9847        drop(held);
9848        submitter.join().unwrap();
9849        assert!(
9850            ctx.queued_update_ids().is_empty(),
9851            "nothing waits to be sent"
9852        );
9853        assert_eq!(client.0.load(Ordering::SeqCst), 0, "nothing is sent");
9854    }
9855
9856    /// A buffered submission that waits at the buffer lock while a write
9857    /// fails is rejected: its checkpoint permit is refused under that lock.
9858    #[test]
9859    fn buffered_submission_waiting_at_the_buffer_lock_sees_the_latch() {
9860        submission_racing_a_failed_write(
9861            CheckpointCoalescer::new(Duration::from_secs(60)),
9862            crate::coordinator::race_points::RacePoint::BufferLock,
9863            |ctx| Box::new(ctx.inner.coalescer.hold_buffer_lock()),
9864        );
9865    }
9866
9867    /// After a failed write, a join is refused its checkpoint permit, so
9868    /// nothing joins the buffer.
9869    #[tokio::test(start_paused = true)]
9870    async fn join_after_a_failed_write_is_refused_its_permit() {
9871        let client = Arc::new(FailFirst::default());
9872        let ctx = DurableContext::new_root_with_client_and_coalescer(
9873            "arn:test".to_owned(),
9874            lambda_runtime::Context::default(),
9875            Arc::new(CheckpointLog::empty()),
9876            Arc::clone(&client) as Arc<dyn ExecutionClient>,
9877            "t0".to_owned(),
9878            CheckpointCoalescer::new(Duration::from_secs(60)),
9879        );
9880        ctx.coordinator().fail_write(
9881            FatalError {
9882                error_type: "Failed".to_owned(),
9883                error_message: "failed".to_owned(),
9884                severity: FatalSeverity::FailExecution,
9885            },
9886            || {},
9887        );
9888        assert!(
9889            ctx.coordinator().acquire(PermitKind::Checkpoint).is_none(),
9890            "no checkpoint permit"
9891        );
9892        assert!(
9893            ctx.coordinator().acquire(PermitKind::Activity).is_some(),
9894            "other permits are still taken"
9895        );
9896        drop(ctx.submit_checkpoint_updates(vec![update("b", OperationAction::Start)]));
9897        assert_eq!(ctx.queued_update_ids(), Vec::<String>::new());
9898    }
9899
9900    /// The end-of-invocation flush stops at a withdrawn batch: its
9901    /// contributor never resolves, and the flush reports no failure.
9902    #[tokio::test(start_paused = true)]
9903    async fn end_flush_stops_at_withdrawal() {
9904        let client = Arc::new(Tokenless::default());
9905        let ctx = ctx_on(
9906            &client,
9907            Some(CheckpointCoalescer::new(Duration::from_secs(60))),
9908        );
9909        let write = ctx.submit_checkpoint_updates(vec![update("a", OperationAction::Start)]);
9910        ctx.flush_pending_checkpoints().await;
9911        assert!(ctx.coordinator().withdrawn());
9912        assert_eq!(client.0.load(Ordering::SeqCst), 1);
9913        assert!(
9914            tokio::time::timeout(Duration::from_secs(120), write)
9915                .await
9916                .is_err(),
9917            "the contributor never resolves"
9918        );
9919        assert_eq!(
9920            client.0.load(Ordering::SeqCst),
9921            1,
9922            "the window flush sends nothing"
9923        );
9924    }
9925}