Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    pin::Pin,
8    sync::{
9        atomic::{AtomicBool, Ordering},
10        Arc, Mutex, OnceLock,
11    },
12    task::{Context as TaskContext, Poll},
13    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
14};
15
16use apache_avro::{from_avro_datum, from_value, to_avro_datum, to_value, Schema};
17use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
18use futures_util::{future::OptionFuture, task::noop_waker_ref};
19use serde::{de::DeserializeOwned, Deserialize, Serialize};
20pub use serde_json::{json, Value};
21use thiserror::Error;
22
23pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
24pub const CONTROL_PLANE_VERSION: &str = "2";
25pub const DEFAULT_CODEC: &str = "avro";
26pub const JSON_CODEC: &str = "json";
27pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
28/// Worker-registration capability for server-routed read-only queries.
29pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
30/// First additive worker protocol that defines query-task transport.
31pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
32
33const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
34const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
35    "Workflow task waiting for scheduled history.";
36const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
37
38const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
39    "lease_expired",
40    "query_task_not_found",
41    "query_task_not_leased",
42    "query_task_timed_out",
43];
44
45const AVRO_PAYLOAD_SCHEMA_JSON: &str = r#"{"type":"record","name":"Payload","namespace":"durable_workflow","fields":[{"name":"json","type":"string"},{"name":"version","type":"int","default":1}]}"#;
46const AVRO_PAYLOAD_VERSION: i32 = 1;
47
48static AVRO_PAYLOAD_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
49
50#[derive(Clone, Copy)]
51enum RequestProtocol {
52    ControlPlane,
53    Worker(&'static str),
54}
55
56impl RequestProtocol {
57    fn is_worker(self) -> bool {
58        matches!(self, Self::Worker(_))
59    }
60}
61
62pub type Result<T> = std::result::Result<T, Error>;
63
64#[derive(Debug, Error)]
65pub enum Error {
66    #[error("transport error: {0}")]
67    Transport(#[from] reqwest::Error),
68    #[error("json error: {0}")]
69    Json(#[from] serde_json::Error),
70    #[error("http {status}: {body}")]
71    Http {
72        status: reqwest::StatusCode,
73        body: String,
74    },
75    #[error("codec error: {0}")]
76    Codec(String),
77    #[error(transparent)]
78    QueryFailed(QueryFailure),
79    #[error(transparent)]
80    Protocol(ProtocolFailure),
81    #[error(transparent)]
82    NonDeterministicReplay(ReplayFailure),
83    #[error(transparent)]
84    ChildWorkflowFailed(ChildWorkflowFailure),
85    #[error(transparent)]
86    ActivityFailed(ActivityFailure),
87    #[error(transparent)]
88    WorkflowCommandRejected(WorkflowCommandRejection),
89    #[error(transparent)]
90    WorkflowFailed(WorkflowTerminalOutcome),
91    #[error(transparent)]
92    WorkflowCancelled(WorkflowTerminalOutcome),
93    #[error(transparent)]
94    WorkflowTerminated(WorkflowTerminalOutcome),
95    #[error(transparent)]
96    WorkflowTimedOut(WorkflowTerminalOutcome),
97    #[error(transparent)]
98    ActivityTaskRejected(ActivityTaskRejection),
99    #[error("workflow handler {0:?} is not registered")]
100    WorkflowNotRegistered(String),
101    #[error("activity handler {0:?} is not registered")]
102    ActivityNotRegistered(String),
103    #[error("workflow future yielded without emitting a durable command")]
104    WorkflowYieldedWithoutCommand,
105    #[error("workflow state lock is poisoned")]
106    WorkflowStatePoisoned,
107    #[error("timer duration is too large for the worker protocol")]
108    TimerDurationOverflow,
109    #[error("operation timed out")]
110    Timeout,
111    #[error("worker loop error: {0}")]
112    WorkerLoop(String),
113    #[error("invalid child workflow options: {0}")]
114    InvalidChildWorkflowOptions(String),
115    #[error(transparent)]
116    InvalidActivityOptions(ActivityOptionsError),
117}
118
119/// The lifecycle command sent to a workflow execution.
120#[derive(Clone, Copy, Debug, PartialEq, Eq)]
121pub enum WorkflowCommandKind {
122    Cancel,
123    Terminate,
124}
125
126impl WorkflowCommandKind {
127    fn as_str(self) -> &'static str {
128        match self {
129            Self::Cancel => "cancel",
130            Self::Terminate => "terminate",
131        }
132    }
133}
134
135/// Optional structured fields for a cancellation or termination request.
136#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
137pub struct WorkflowCommandOptions {
138    #[serde(skip_serializing_if = "Option::is_none")]
139    pub reason: Option<String>,
140    #[serde(skip_serializing_if = "Option::is_none")]
141    pub request_id: Option<String>,
142}
143
144impl WorkflowCommandOptions {
145    pub fn new() -> Self {
146        Self::default()
147    }
148
149    pub fn reason(mut self, reason: impl Into<String>) -> Self {
150        self.reason = Some(reason.into());
151        self
152    }
153
154    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
155        self.request_id = Some(request_id.into());
156        self
157    }
158}
159
160/// The accepted, machine-readable result of a lifecycle command.
161#[derive(Clone, Debug, PartialEq)]
162pub struct WorkflowCommandResult {
163    pub command: WorkflowCommandKind,
164    pub workflow_id: String,
165    pub run_id: Option<String>,
166    pub outcome: Option<String>,
167    pub reason: Option<String>,
168    pub command_status: Option<String>,
169    pub raw: Value,
170}
171
172/// A stable rejection returned by instance- or selected-run lifecycle commands.
173#[derive(Clone, Debug, Error)]
174#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
175pub struct WorkflowCommandRejection {
176    pub command: WorkflowCommandKind,
177    pub status: u16,
178    pub reason: String,
179    pub message: String,
180    pub workflow_id: String,
181    pub run_id: Option<String>,
182    pub target_scope: Option<String>,
183    pub body: Value,
184}
185
186/// Stable terminal categories returned by [`WorkflowHandle::result`].
187#[derive(Clone, Copy, Debug, PartialEq, Eq)]
188pub enum WorkflowTerminalKind {
189    Failed,
190    Cancelled,
191    Terminated,
192    TimedOut,
193}
194
195/// A typed terminal workflow outcome with durable identity and failure metadata.
196///
197/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
198/// of parsing its display representation. Fields remain `None` when an older
199/// server did not publish that metadata.
200#[derive(Clone, Debug, Error)]
201#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
202pub struct WorkflowTerminalOutcome {
203    pub kind: WorkflowTerminalKind,
204    pub workflow_id: String,
205    pub run_id: Option<String>,
206    pub reason: String,
207    pub failure_category: Option<String>,
208    pub failure_id: Option<String>,
209    pub exception_type: Option<String>,
210    pub exception_class: Option<String>,
211    pub non_retryable: Option<bool>,
212    pub message: Option<String>,
213    pub exception: Option<Value>,
214    pub raw: Value,
215}
216
217/// A worker-side activity settlement or heartbeat rejected by durable state.
218#[derive(Clone, Debug, Error)]
219#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
220pub struct ActivityTaskRejection {
221    pub operation: String,
222    pub status: u16,
223    pub reason: String,
224    pub task_id: String,
225    pub activity_attempt_id: String,
226    pub cancel_requested: bool,
227    pub can_continue: Option<bool>,
228    pub run_closed_reason: Option<String>,
229    pub body: Value,
230}
231
232/// Stable validation categories for [`ActivityOptions`].
233#[derive(Clone, Copy, Debug, PartialEq, Eq)]
234pub enum ActivityOptionsErrorKind {
235    EmptyTaskQueue,
236    EmptyRetryPolicy,
237    InvalidMaxAttempts,
238    BackoffWithoutRetryBudget,
239    TooManyBackoffIntervals,
240    InvalidBackoffCoefficient,
241    BackoffGenerationTooLarge,
242    BackoffOverflow,
243    EmptyNonRetryableErrorType,
244    TimeoutNotPositive,
245    TimeoutOverflow,
246    TimeoutOrder,
247}
248
249/// A machine-readable activity-options validation failure.
250#[derive(Clone, Debug, Error, PartialEq, Eq)]
251#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
252pub struct ActivityOptionsError {
253    pub kind: ActivityOptionsErrorKind,
254    pub field: Option<&'static str>,
255    pub message: String,
256}
257
258impl ActivityOptionsError {
259    fn new(
260        kind: ActivityOptionsErrorKind,
261        field: Option<&'static str>,
262        message: impl Into<String>,
263    ) -> Self {
264        Self {
265            kind,
266            field,
267            message: message.into(),
268        }
269    }
270}
271
272/// Stable terminal categories returned when an awaited activity does not succeed.
273#[derive(Clone, Copy, Debug, PartialEq, Eq)]
274pub enum ActivityFailureKind {
275    Failed,
276    Cancelled,
277    TimedOut,
278}
279
280/// A stable, machine-readable terminal activity failure.
281///
282/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
283/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
284#[derive(Clone, Debug, Error)]
285#[error("activity failed ({reason}): {message}")]
286pub struct ActivityFailure {
287    pub kind: ActivityFailureKind,
288    pub reason: String,
289    pub message: String,
290    pub activity_execution_id: Option<String>,
291    pub activity_attempt_id: Option<String>,
292    pub activity_type: Option<String>,
293    pub activity_class: Option<String>,
294    pub attempt_number: Option<u64>,
295    pub failure_id: Option<String>,
296    pub failure_category: Option<String>,
297    pub timeout_kind: Option<String>,
298    pub non_retryable: bool,
299    pub exception_type: Option<String>,
300    pub exception_class: Option<String>,
301    pub code: Option<Value>,
302    pub exception: Option<Value>,
303}
304
305/// Stable terminal categories returned when an awaited child does not succeed.
306#[derive(Clone, Copy, Debug, PartialEq, Eq)]
307pub enum ChildWorkflowFailureKind {
308    Failed,
309    Cancelled,
310    Terminated,
311}
312
313/// A stable, machine-readable child workflow failure delivered to its parent.
314///
315/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
316/// of parsing the display message. Child and parent identifiers retain the
317/// relationship recorded in durable history across worker restarts.
318#[derive(Clone, Debug, Error)]
319#[error("child workflow failed ({reason}): {message}")]
320pub struct ChildWorkflowFailure {
321    pub kind: ChildWorkflowFailureKind,
322    pub reason: String,
323    pub message: String,
324    pub parent_workflow_id: Option<String>,
325    pub parent_workflow_run_id: Option<String>,
326    pub child_workflow_id: Option<String>,
327    pub child_workflow_run_id: Option<String>,
328    pub child_workflow_type: Option<String>,
329    pub failure_id: Option<String>,
330    pub failure_category: Option<String>,
331    pub exception_type: Option<String>,
332    pub exception_class: Option<String>,
333    pub non_retryable: bool,
334    pub code: Option<Value>,
335    pub exception: Option<Value>,
336}
337
338/// The identity of one durable workflow execution.
339#[derive(Clone, Debug, PartialEq, Eq)]
340pub struct WorkflowIdentity {
341    pub workflow_id: Option<String>,
342    pub run_id: Option<String>,
343}
344
345/// A successful child result together with its durable parent-child identity.
346#[derive(Clone, Debug, PartialEq)]
347pub struct ChildWorkflowResult {
348    pub parent: WorkflowIdentity,
349    pub child: WorkflowIdentity,
350    pub child_workflow_type: Option<String>,
351    pub result: Value,
352}
353
354/// Server behavior when a parent closes while its child is still open.
355#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
356pub enum ParentClosePolicy {
357    #[default]
358    Abandon,
359    RequestCancel,
360    Terminate,
361}
362
363impl ParentClosePolicy {
364    fn as_str(self) -> &'static str {
365        match self {
366            Self::Abandon => "abandon",
367            Self::RequestCancel => "request_cancel",
368            Self::Terminate => "terminate",
369        }
370    }
371}
372
373/// Durable retry policy for one child workflow invocation.
374#[derive(Clone, Debug, Default, PartialEq, Eq)]
375pub struct ChildWorkflowRetryPolicy {
376    pub max_attempts: Option<u32>,
377    pub backoff_seconds: Vec<u64>,
378    pub non_retryable_error_types: Vec<String>,
379}
380
381/// Options recorded with a child-workflow command.
382///
383/// The task queue is mandatory so routing is explicit and replay-stable.
384#[derive(Clone, Debug, PartialEq, Eq)]
385pub struct ChildWorkflowOptions {
386    pub task_queue: String,
387    pub parent_close_policy: ParentClosePolicy,
388    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
389    pub execution_timeout_seconds: Option<u64>,
390    pub run_timeout_seconds: Option<u64>,
391}
392
393impl ChildWorkflowOptions {
394    pub fn new(task_queue: impl Into<String>) -> Self {
395        Self {
396            task_queue: task_queue.into(),
397            parent_close_policy: ParentClosePolicy::Abandon,
398            retry_policy: None,
399            execution_timeout_seconds: None,
400            run_timeout_seconds: None,
401        }
402    }
403
404    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
405        self.parent_close_policy = policy;
406        self
407    }
408
409    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
410        self.retry_policy = Some(policy);
411        self
412    }
413
414    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
415        self.execution_timeout_seconds = Some(seconds);
416        self
417    }
418
419    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
420        self.run_timeout_seconds = Some(seconds);
421        self
422    }
423}
424
425/// Backoff intervals for one durable activity retry policy.
426#[derive(Clone, Debug, PartialEq, Eq)]
427pub enum ActivityBackoff {
428    /// Use these intervals between attempts. The server repeats the final
429    /// interval if the retry budget contains more attempts than entries.
430    Explicit(Vec<Duration>),
431    /// Generate one interval for every retry using integer exponential growth.
432    Exponential {
433        initial_interval: Duration,
434        coefficient: u32,
435        maximum_interval: Option<Duration>,
436    },
437}
438
439/// Durable server-side retry policy for one activity execution.
440#[derive(Clone, Debug, Default, PartialEq, Eq)]
441pub struct ActivityRetryPolicy {
442    pub max_attempts: Option<u32>,
443    pub backoff: Option<ActivityBackoff>,
444    pub non_retryable_error_types: Vec<String>,
445}
446
447impl ActivityRetryPolicy {
448    /// Start a policy with a finite attempt budget, including the first attempt.
449    pub fn new(max_attempts: u32) -> Self {
450        Self {
451            max_attempts: Some(max_attempts),
452            ..Self::default()
453        }
454    }
455
456    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
457        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
458        self
459    }
460
461    pub fn exponential_backoff(
462        mut self,
463        initial_interval: Duration,
464        coefficient: u32,
465        maximum_interval: Option<Duration>,
466    ) -> Self {
467        self.backoff = Some(ActivityBackoff::Exponential {
468            initial_interval,
469            coefficient,
470            maximum_interval,
471        });
472        self
473    }
474
475    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
476        self.non_retryable_error_types.push(error_type.into());
477        self
478    }
479
480    pub fn non_retryable_error_types(
481        mut self,
482        error_types: impl IntoIterator<Item = impl Into<String>>,
483    ) -> Self {
484        self.non_retryable_error_types
485            .extend(error_types.into_iter().map(Into::into));
486        self
487    }
488}
489
490/// Options recorded atomically on one deterministic `schedule_activity` command.
491///
492/// Durations are rounded up to whole seconds when encoded, so the server never
493/// receives a shorter timeout or backoff than the caller requested.
494#[derive(Clone, Debug, Default, PartialEq, Eq)]
495pub struct ActivityOptions {
496    pub task_queue: Option<String>,
497    pub retry_policy: Option<ActivityRetryPolicy>,
498    pub start_to_close_timeout: Option<Duration>,
499    pub schedule_to_start_timeout: Option<Duration>,
500    pub schedule_to_close_timeout: Option<Duration>,
501    pub heartbeat_timeout: Option<Duration>,
502}
503
504impl ActivityOptions {
505    pub fn new() -> Self {
506        Self::default()
507    }
508
509    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
510        self.task_queue = Some(task_queue.into());
511        self
512    }
513
514    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
515        self.retry_policy = Some(policy);
516        self
517    }
518
519    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
520        self.start_to_close_timeout = Some(timeout);
521        self
522    }
523
524    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
525        self.schedule_to_start_timeout = Some(timeout);
526        self
527    }
528
529    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
530        self.schedule_to_close_timeout = Some(timeout);
531        self
532    }
533
534    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
535        self.heartbeat_timeout = Some(timeout);
536        self
537    }
538
539    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
540        if self
541            .task_queue
542            .as_deref()
543            .is_some_and(|queue| queue.trim().is_empty())
544        {
545            return Err(ActivityOptionsError::new(
546                ActivityOptionsErrorKind::EmptyTaskQueue,
547                Some("task_queue"),
548                "task_queue must not be empty",
549            ));
550        }
551
552        for (field, value) in [
553            ("start_to_close_timeout", self.start_to_close_timeout),
554            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
555            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
556            ("heartbeat_timeout", self.heartbeat_timeout),
557        ] {
558            if value.is_some_and(|value| value.is_zero()) {
559                return Err(ActivityOptionsError::new(
560                    ActivityOptionsErrorKind::TimeoutNotPositive,
561                    Some(field),
562                    format!("{field} must be positive"),
563                ));
564            }
565        }
566
567        validate_timeout_order(
568            "heartbeat_timeout",
569            self.heartbeat_timeout,
570            "start_to_close_timeout",
571            self.start_to_close_timeout,
572        )?;
573        validate_timeout_order(
574            "start_to_close_timeout",
575            self.start_to_close_timeout,
576            "schedule_to_close_timeout",
577            self.schedule_to_close_timeout,
578        )?;
579        validate_timeout_order(
580            "schedule_to_start_timeout",
581            self.schedule_to_start_timeout,
582            "schedule_to_close_timeout",
583            self.schedule_to_close_timeout,
584        )?;
585
586        Ok(ValidatedActivityOptions {
587            task_queue: self.task_queue.clone(),
588            retry_policy: self
589                .retry_policy
590                .as_ref()
591                .map(validate_activity_retry_policy)
592                .transpose()?,
593            start_to_close_timeout: timeout_seconds(
594                "start_to_close_timeout",
595                self.start_to_close_timeout,
596            )?,
597            schedule_to_start_timeout: timeout_seconds(
598                "schedule_to_start_timeout",
599                self.schedule_to_start_timeout,
600            )?,
601            schedule_to_close_timeout: timeout_seconds(
602                "schedule_to_close_timeout",
603                self.schedule_to_close_timeout,
604            )?,
605            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
606        })
607    }
608}
609
610#[derive(Clone, Debug)]
611struct ValidatedActivityOptions {
612    task_queue: Option<String>,
613    retry_policy: Option<Value>,
614    start_to_close_timeout: Option<u64>,
615    schedule_to_start_timeout: Option<u64>,
616    schedule_to_close_timeout: Option<u64>,
617    heartbeat_timeout: Option<u64>,
618}
619
620fn validate_timeout_order(
621    smaller_name: &'static str,
622    smaller: Option<Duration>,
623    larger_name: &'static str,
624    larger: Option<Duration>,
625) -> std::result::Result<(), ActivityOptionsError> {
626    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
627        return Err(ActivityOptionsError::new(
628            ActivityOptionsErrorKind::TimeoutOrder,
629            Some(smaller_name),
630            format!("{smaller_name} must be <= {larger_name}"),
631        ));
632    }
633    Ok(())
634}
635
636fn timeout_seconds(
637    field: &'static str,
638    value: Option<Duration>,
639) -> std::result::Result<Option<u64>, ActivityOptionsError> {
640    value
641        .map(|value| {
642            activity_protocol_seconds(value).ok_or_else(|| {
643                ActivityOptionsError::new(
644                    ActivityOptionsErrorKind::TimeoutOverflow,
645                    Some(field),
646                    format!("{field} is too large for the worker protocol"),
647                )
648            })
649        })
650        .transpose()
651}
652
653fn duration_seconds_ceil(value: Duration) -> Option<u64> {
654    value
655        .as_secs()
656        .checked_add(u64::from(value.subsec_nanos() > 0))
657}
658
659fn activity_protocol_seconds(value: Duration) -> Option<u64> {
660    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
661}
662
663fn validate_activity_retry_policy(
664    policy: &ActivityRetryPolicy,
665) -> std::result::Result<Value, ActivityOptionsError> {
666    if policy.max_attempts.is_none()
667        && policy.backoff.is_none()
668        && policy.non_retryable_error_types.is_empty()
669    {
670        return Err(ActivityOptionsError::new(
671            ActivityOptionsErrorKind::EmptyRetryPolicy,
672            Some("retry_policy"),
673            "retry_policy must configure at least one field",
674        ));
675    }
676    if policy.max_attempts == Some(0) {
677        return Err(ActivityOptionsError::new(
678            ActivityOptionsErrorKind::InvalidMaxAttempts,
679            Some("retry_policy.max_attempts"),
680            "max_attempts must be >= 1",
681        ));
682    }
683    if policy
684        .non_retryable_error_types
685        .iter()
686        .any(|error_type| error_type.trim().is_empty())
687    {
688        return Err(ActivityOptionsError::new(
689            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
690            Some("retry_policy.non_retryable_error_types"),
691            "non_retryable_error_types must not contain empty values",
692        ));
693    }
694
695    let backoff_seconds = match &policy.backoff {
696        None => None,
697        Some(backoff) => {
698            let max_attempts = policy.max_attempts.ok_or_else(|| {
699                ActivityOptionsError::new(
700                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
701                    Some("retry_policy.backoff"),
702                    "backoff requires max_attempts",
703                )
704            })?;
705            let retry_count = max_attempts.saturating_sub(1) as usize;
706            let intervals = match backoff {
707                ActivityBackoff::Explicit(intervals) => {
708                    if intervals.len() > retry_count {
709                        return Err(ActivityOptionsError::new(
710                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
711                            Some("retry_policy.backoff"),
712                            "backoff interval count must not exceed max_attempts - 1",
713                        ));
714                    }
715                    intervals.clone()
716                }
717                ActivityBackoff::Exponential {
718                    initial_interval,
719                    coefficient,
720                    maximum_interval,
721                } => {
722                    if *coefficient < 1 {
723                        return Err(ActivityOptionsError::new(
724                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
725                            Some("retry_policy.backoff.coefficient"),
726                            "backoff coefficient must be >= 1",
727                        ));
728                    }
729                    if retry_count > 10_000 {
730                        return Err(ActivityOptionsError::new(
731                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
732                            Some("retry_policy.max_attempts"),
733                            "generated backoff supports at most 10000 retry intervals",
734                        ));
735                    }
736                    let mut current = *initial_interval;
737                    let mut intervals = Vec::with_capacity(retry_count);
738                    for _ in 0..retry_count {
739                        let interval = maximum_interval
740                            .map(|maximum| current.min(maximum))
741                            .unwrap_or(current);
742                        intervals.push(interval);
743                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
744                            break;
745                        }
746                        current = current.checked_mul(*coefficient).ok_or_else(|| {
747                            ActivityOptionsError::new(
748                                ActivityOptionsErrorKind::BackoffOverflow,
749                                Some("retry_policy.backoff"),
750                                "generated backoff interval overflowed",
751                            )
752                        })?;
753                    }
754                    intervals
755                }
756            };
757            Some(
758                intervals
759                    .into_iter()
760                    .map(|interval| {
761                        activity_protocol_seconds(interval).ok_or_else(|| {
762                            ActivityOptionsError::new(
763                                ActivityOptionsErrorKind::BackoffOverflow,
764                                Some("retry_policy.backoff"),
765                                "backoff interval is too large for the worker protocol",
766                            )
767                        })
768                    })
769                    .collect::<std::result::Result<Vec<_>, _>>()?,
770            )
771        }
772    };
773
774    let mut encoded = serde_json::Map::new();
775    if let Some(max_attempts) = policy.max_attempts {
776        encoded.insert("max_attempts".to_string(), json!(max_attempts));
777    }
778    if let Some(backoff_seconds) = backoff_seconds {
779        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
780    }
781    if !policy.non_retryable_error_types.is_empty() {
782        let mut canonical_error_types = Vec::new();
783        for error_type in policy
784            .non_retryable_error_types
785            .iter()
786            .map(|error_type| error_type.trim())
787        {
788            if !canonical_error_types.contains(&error_type) {
789                canonical_error_types.push(error_type);
790            }
791        }
792        encoded.insert(
793            "non_retryable_error_types".to_string(),
794            json!(canonical_error_types),
795        );
796    }
797    Ok(Value::Object(encoded))
798}
799
800/// A stable, machine-readable failure raised when workflow code no longer
801/// reconstructs the durable command stream recorded in history.
802#[derive(Clone, Debug, Error)]
803#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
804pub struct ReplayFailure {
805    pub reason: String,
806    pub sequence: Option<u64>,
807    pub expected: Option<String>,
808    pub actual: Option<String>,
809    pub message: String,
810}
811
812impl ReplayFailure {
813    fn new(
814        reason: impl Into<String>,
815        sequence: Option<u64>,
816        expected: Option<String>,
817        actual: Option<String>,
818        message: impl Into<String>,
819    ) -> Self {
820        Self {
821            reason: reason.into(),
822            sequence,
823            expected,
824            actual,
825            message: message.into(),
826        }
827    }
828}
829
830/// A stable, machine-readable workflow query or query-task settlement failure.
831#[derive(Clone, Debug, Error)]
832#[error("query failed ({reason}, HTTP {status}): {message}")]
833pub struct QueryFailure {
834    pub status: u16,
835    pub reason: String,
836    pub message: String,
837    pub body: Value,
838}
839
840/// A stable failure returned when a server rejects an SDK protocol version.
841#[derive(Clone, Debug, Error)]
842#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
843pub struct ProtocolFailure {
844    pub status: u16,
845    pub reason: String,
846    pub message: String,
847    pub supported_version: Option<String>,
848    pub requested_version: Option<String>,
849    pub body: Value,
850}
851
852#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
853pub struct PayloadEnvelope {
854    pub codec: String,
855    pub blob: String,
856}
857
858impl PayloadEnvelope {
859    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
860        encode_payload(value, DEFAULT_CODEC)
861    }
862
863    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
864        encode_payload(value, JSON_CODEC)
865    }
866}
867
868pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
869    let value = serde_json::to_value(value)?;
870    let blob = encode_value_blob(&value, codec)?;
871
872    Ok(PayloadEnvelope {
873        codec: codec.to_string(),
874        blob,
875    })
876}
877
878pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
879    let value = decode_blob(&envelope.blob, &envelope.codec)?;
880    Ok(serde_json::from_value(value)?)
881}
882
883fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
884    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
885}
886
887fn encode_value_blob(value: &Value, codec: &str) -> Result<String> {
888    match codec {
889        JSON_CODEC => Ok(serde_json::to_string(value)?),
890        DEFAULT_CODEC => encode_avro_generic(value),
891        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
892    }
893}
894
895fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
896    if value.is_null() {
897        return Ok(Value::Null);
898    }
899
900    if let Some(object) = value.as_object() {
901        if let (Some(codec), Some(blob)) = (
902            object.get("codec").and_then(Value::as_str),
903            object.get("blob").and_then(Value::as_str),
904        ) {
905            return decode_blob(blob, codec);
906        }
907    }
908
909    if let Some(blob) = value.as_str() {
910        return decode_blob(blob, fallback_codec);
911    }
912
913    Ok(value.clone())
914}
915
916fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
917    match codec {
918        JSON_CODEC => Ok(serde_json::from_str(blob)?),
919        DEFAULT_CODEC => decode_avro_generic(blob),
920        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
921    }
922}
923
924fn encode_avro_generic(value: &Value) -> Result<String> {
925    let json = serde_json::to_string(value)?;
926    let datum = to_value(AvroPayload {
927        json,
928        version: AVRO_PAYLOAD_VERSION,
929    })
930    .map_err(|err| Error::Codec(format!("could not convert avro generic wrapper: {err}")))?;
931    let datum = to_avro_datum(avro_payload_schema()?, datum)
932        .map_err(|err| Error::Codec(format!("could not encode avro generic wrapper: {err}")))?;
933
934    let mut bytes = Vec::with_capacity(datum.len() + 1);
935    bytes.push(0x00);
936    bytes.extend_from_slice(&datum);
937    Ok(BASE64.encode(bytes))
938}
939
940fn decode_avro_generic(blob: &str) -> Result<Value> {
941    let bytes = BASE64
942        .decode(blob)
943        .map_err(|err| Error::Codec(format!("invalid avro base64 payload: {err}")))?;
944
945    if bytes.is_empty() {
946        return Err(Error::Codec("avro payload is empty".to_string()));
947    }
948
949    match bytes[0] {
950        0x00 => {}
951        0x01 => {
952            return Err(Error::Codec(
953                "typed avro payloads require a schema context; v1 supports the generic wrapper"
954                    .to_string(),
955            ));
956        }
957        other => {
958            return Err(Error::Codec(format!(
959                "unknown avro payload prefix 0x{other:02x}"
960            )));
961        }
962    }
963
964    let mut datum = &bytes[1..];
965    let datum = from_avro_datum(avro_payload_schema()?, &mut datum, None)
966        .map_err(|err| Error::Codec(format!("could not decode avro generic wrapper: {err}")))?;
967    let payload: AvroPayload = from_value(&datum)
968        .map_err(|err| Error::Codec(format!("invalid avro generic wrapper record: {err}")))?;
969
970    if payload.version != AVRO_PAYLOAD_VERSION {
971        return Err(Error::Codec(format!(
972            "unsupported avro generic wrapper version {}",
973            payload.version
974        )));
975    }
976
977    Ok(serde_json::from_str(&payload.json)?)
978}
979
980#[derive(Debug, Serialize, Deserialize)]
981struct AvroPayload {
982    json: String,
983    version: i32,
984}
985
986fn avro_payload_schema() -> Result<&'static Schema> {
987    match AVRO_PAYLOAD_SCHEMA.get_or_init(|| {
988        Schema::parse_str(AVRO_PAYLOAD_SCHEMA_JSON)
989            .map_err(|err| format!("could not parse avro payload schema: {err}"))
990    }) {
991        Ok(schema) => Ok(schema),
992        Err(message) => Err(Error::Codec(message.clone())),
993    }
994}
995
996#[derive(Clone, Debug)]
997pub struct Client {
998    http: reqwest::Client,
999    base_url: String,
1000    token: Option<String>,
1001    control_token: Option<String>,
1002    worker_token: Option<String>,
1003    namespace: String,
1004}
1005
1006impl Client {
1007    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1008        Self::builder(base_url).build()
1009    }
1010
1011    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1012        ClientBuilder {
1013            base_url: base_url.into(),
1014            token: None,
1015            control_token: None,
1016            worker_token: None,
1017            namespace: "default".to_string(),
1018            timeout: Duration::from_secs(60),
1019        }
1020    }
1021
1022    pub async fn health(&self) -> Result<Value> {
1023        self.request_json(
1024            reqwest::Method::GET,
1025            "/health",
1026            RequestProtocol::ControlPlane,
1027            Option::<&Value>::None,
1028        )
1029        .await
1030    }
1031
1032    pub async fn cluster_info(&self) -> Result<Value> {
1033        self.request_json(
1034            reqwest::Method::GET,
1035            "/cluster/info",
1036            RequestProtocol::ControlPlane,
1037            Option::<&Value>::None,
1038        )
1039        .await
1040    }
1041
1042    pub async fn start_workflow<T: Serialize>(
1043        &self,
1044        workflow_type: &str,
1045        task_queue: &str,
1046        workflow_id: &str,
1047        input: T,
1048    ) -> Result<WorkflowHandle> {
1049        let input = serde_json::to_value(input)?;
1050        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1051        let body = json!({
1052            "workflow_id": workflow_id,
1053            "workflow_type": workflow_type,
1054            "task_queue": task_queue,
1055            "input": input_envelope,
1056            "execution_timeout_seconds": 3600,
1057            "run_timeout_seconds": 600
1058        });
1059
1060        let data: Value = self
1061            .request_json(
1062                reqwest::Method::POST,
1063                "/workflows",
1064                RequestProtocol::ControlPlane,
1065                Some(&body),
1066            )
1067            .await?;
1068
1069        Ok(WorkflowHandle {
1070            client: self.clone(),
1071            workflow_id: data
1072                .get("workflow_id")
1073                .and_then(Value::as_str)
1074                .unwrap_or(workflow_id)
1075                .to_string(),
1076            run_id: data
1077                .get("run_id")
1078                .and_then(Value::as_str)
1079                .map(str::to_string),
1080            workflow_type: data
1081                .get("workflow_type")
1082                .and_then(Value::as_str)
1083                .unwrap_or(workflow_type)
1084                .to_string(),
1085        })
1086    }
1087
1088    pub async fn signal_workflow<T: Serialize>(
1089        &self,
1090        workflow_id: &str,
1091        signal_name: &str,
1092        input: T,
1093    ) -> Result<Value> {
1094        let input = serde_json::to_value(input)?;
1095        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1096        let body = json!({
1097            "input": input_envelope
1098        });
1099        let path = format!("/workflows/{workflow_id}/signal/{signal_name}");
1100        self.request_json(
1101            reqwest::Method::POST,
1102            &path,
1103            RequestProtocol::ControlPlane,
1104            Some(&body),
1105        )
1106        .await
1107    }
1108
1109    /// Request cooperative cancellation of the current run for an instance.
1110    pub async fn cancel_workflow(
1111        &self,
1112        workflow_id: &str,
1113        options: WorkflowCommandOptions,
1114    ) -> Result<WorkflowCommandResult> {
1115        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1116            .await
1117    }
1118
1119    /// Request cooperative cancellation only if `run_id` is still current.
1120    pub async fn cancel_workflow_run(
1121        &self,
1122        workflow_id: &str,
1123        run_id: &str,
1124        options: WorkflowCommandOptions,
1125    ) -> Result<WorkflowCommandResult> {
1126        self.workflow_command(
1127            workflow_id,
1128            Some(run_id),
1129            WorkflowCommandKind::Cancel,
1130            options,
1131        )
1132        .await
1133    }
1134
1135    /// Forcefully terminate the current run for an instance.
1136    pub async fn terminate_workflow(
1137        &self,
1138        workflow_id: &str,
1139        options: WorkflowCommandOptions,
1140    ) -> Result<WorkflowCommandResult> {
1141        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1142            .await
1143    }
1144
1145    /// Forcefully terminate only if `run_id` is still current.
1146    pub async fn terminate_workflow_run(
1147        &self,
1148        workflow_id: &str,
1149        run_id: &str,
1150        options: WorkflowCommandOptions,
1151    ) -> Result<WorkflowCommandResult> {
1152        self.workflow_command(
1153            workflow_id,
1154            Some(run_id),
1155            WorkflowCommandKind::Terminate,
1156            options,
1157        )
1158        .await
1159    }
1160
1161    async fn workflow_command(
1162        &self,
1163        workflow_id: &str,
1164        run_id: Option<&str>,
1165        command: WorkflowCommandKind,
1166        options: WorkflowCommandOptions,
1167    ) -> Result<WorkflowCommandResult> {
1168        let path = match run_id {
1169            Some(run_id) => format!(
1170                "/workflows/{workflow_id}/runs/{run_id}/{}",
1171                command.as_str()
1172            ),
1173            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1174        };
1175        let data = match self
1176            .request_json(
1177                reqwest::Method::POST,
1178                &path,
1179                RequestProtocol::ControlPlane,
1180                Some(&options),
1181            )
1182            .await
1183        {
1184            Ok(data) => data,
1185            Err(Error::Http { status, body }) => {
1186                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1187                    command,
1188                    status,
1189                    body,
1190                    workflow_id,
1191                    run_id,
1192                )));
1193            }
1194            Err(error) => return Err(error),
1195        };
1196
1197        Ok(workflow_command_result(command, data, workflow_id, run_id))
1198    }
1199
1200    /// Execute a named, read-only query against a running or completed workflow.
1201    ///
1202    /// Arguments and results use the platform payload envelope. Server and
1203    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1204    /// reason, HTTP status, and original response body.
1205    pub async fn query_workflow<T: Serialize>(
1206        &self,
1207        workflow_id: &str,
1208        query_name: &str,
1209        input: T,
1210    ) -> Result<Value> {
1211        let input = serde_json::to_value(input)?;
1212        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1213        let body = json!({
1214            "input": input_envelope
1215        });
1216        let path = format!("/workflows/{workflow_id}/query/{query_name}");
1217        let response: Value = match self
1218            .request_json(
1219                reqwest::Method::POST,
1220                &path,
1221                RequestProtocol::ControlPlane,
1222                Some(&body),
1223            )
1224            .await
1225        {
1226            Ok(response) => response,
1227            Err(Error::Http { status, body }) => {
1228                return Err(Error::QueryFailed(query_failure(status, body)));
1229            }
1230            Err(error) => return Err(error),
1231        };
1232
1233        if let Some(envelope) = response
1234            .get("result_envelope")
1235            .filter(|envelope| !envelope.is_null())
1236        {
1237            return decode_wire_value(envelope, DEFAULT_CODEC);
1238        }
1239
1240        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1241    }
1242
1243    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
1244        let path = format!("/workflows/{workflow_id}");
1245        let mut data: WorkflowDescription = self
1246            .request_json(
1247                reqwest::Method::GET,
1248                &path,
1249                RequestProtocol::ControlPlane,
1250                Option::<&Value>::None,
1251            )
1252            .await?;
1253        data.decode_payloads()?;
1254        Ok(data)
1255    }
1256
1257    /// Describe one selected run, including historical terminal runs.
1258    pub async fn describe_workflow_run(
1259        &self,
1260        workflow_id: &str,
1261        run_id: &str,
1262    ) -> Result<WorkflowDescription> {
1263        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
1264        let mut data: WorkflowDescription = self
1265            .request_json(
1266                reqwest::Method::GET,
1267                &path,
1268                RequestProtocol::ControlPlane,
1269                Option::<&Value>::None,
1270            )
1271            .await?;
1272        data.decode_payloads()?;
1273        Ok(data)
1274    }
1275
1276    pub async fn register_worker(
1277        &self,
1278        worker_id: &str,
1279        task_queue: &str,
1280        supported_workflow_types: Vec<String>,
1281        supported_activity_types: Vec<String>,
1282        max_concurrent_workflow_tasks: usize,
1283        max_concurrent_activity_tasks: usize,
1284    ) -> Result<RegisterWorkerResponse> {
1285        self.register_worker_with_capabilities(
1286            worker_id,
1287            task_queue,
1288            supported_workflow_types,
1289            supported_activity_types,
1290            max_concurrent_workflow_tasks,
1291            max_concurrent_activity_tasks,
1292            Vec::new(),
1293        )
1294        .await
1295    }
1296
1297    /// Register a worker and explicitly advertise additive worker capabilities.
1298    pub async fn register_worker_with_capabilities(
1299        &self,
1300        worker_id: &str,
1301        task_queue: &str,
1302        supported_workflow_types: Vec<String>,
1303        supported_activity_types: Vec<String>,
1304        max_concurrent_workflow_tasks: usize,
1305        max_concurrent_activity_tasks: usize,
1306        capabilities: Vec<String>,
1307    ) -> Result<RegisterWorkerResponse> {
1308        let body = json!({
1309            "worker_id": worker_id,
1310            "task_queue": task_queue,
1311            "runtime": "rust",
1312            "sdk_version": SDK_VERSION,
1313            "supported_workflow_types": supported_workflow_types,
1314            "supported_activity_types": supported_activity_types,
1315            "capabilities": capabilities,
1316            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
1317            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
1318        });
1319
1320        self.request_json(
1321            reqwest::Method::POST,
1322            "/worker/register",
1323            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1324            Some(&body),
1325        )
1326        .await
1327    }
1328
1329    /// Long-poll for an ephemeral, read-only workflow query task.
1330    pub async fn poll_query_task(
1331        &self,
1332        worker_id: &str,
1333        task_queue: &str,
1334        timeout: Duration,
1335    ) -> Result<Option<QueryTask>> {
1336        Ok(self
1337            .poll_query_task_response(worker_id, task_queue, timeout)
1338            .await?
1339            .task)
1340    }
1341
1342    /// Poll a query task while preserving server stop and drain metadata.
1343    pub async fn poll_query_task_response(
1344        &self,
1345        worker_id: &str,
1346        task_queue: &str,
1347        timeout: Duration,
1348    ) -> Result<PollQueryTaskResponse> {
1349        let timeout_seconds = long_poll_timeout_seconds(timeout);
1350        let body = json!({
1351            "worker_id": worker_id,
1352            "task_queue": task_queue,
1353            "poll_request_id": unique_request_id("rust-query-poll"),
1354            "timeout_seconds": timeout_seconds,
1355        });
1356        worker_poll_response(
1357            self.request_json_with_timeout(
1358                reqwest::Method::POST,
1359                "/worker/query-tasks/poll",
1360                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1361                Some(&body),
1362                timeout + Duration::from_secs(5),
1363            )
1364            .await,
1365        )
1366    }
1367
1368    /// Complete a query task without appending workflow history.
1369    pub async fn complete_query_task(
1370        &self,
1371        query_task_id: &str,
1372        lease_owner: &str,
1373        query_task_attempt: u64,
1374        result: Value,
1375        codec: &str,
1376    ) -> Result<Value> {
1377        let result_envelope = encode_value_envelope(&result, codec)?;
1378        self.complete_query_task_with_envelope(
1379            query_task_id,
1380            lease_owner,
1381            query_task_attempt,
1382            result,
1383            result_envelope,
1384        )
1385        .await
1386    }
1387
1388    async fn complete_query_task_with_envelope(
1389        &self,
1390        query_task_id: &str,
1391        lease_owner: &str,
1392        query_task_attempt: u64,
1393        result: Value,
1394        result_envelope: Value,
1395    ) -> Result<Value> {
1396        let body = json!({
1397            "lease_owner": lease_owner,
1398            "query_task_attempt": query_task_attempt,
1399            "result": result,
1400            "result_envelope": result_envelope,
1401        });
1402        let path = format!("/worker/query-tasks/{query_task_id}/complete");
1403        let response = self
1404            .request_json(
1405                reqwest::Method::POST,
1406                &path,
1407                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1408                Some(&body),
1409            )
1410            .await;
1411        query_task_response(response)
1412    }
1413
1414    /// Report a stable machine-readable query-task failure.
1415    pub async fn fail_query_task(
1416        &self,
1417        query_task_id: &str,
1418        lease_owner: &str,
1419        query_task_attempt: u64,
1420        message: impl Into<String>,
1421        reason: impl Into<String>,
1422        failure_type: impl Into<String>,
1423    ) -> Result<Value> {
1424        let body = json!({
1425            "lease_owner": lease_owner,
1426            "query_task_attempt": query_task_attempt,
1427            "failure": {
1428                "message": message.into(),
1429                "reason": reason.into(),
1430                "type": failure_type.into(),
1431            }
1432        });
1433        let path = format!("/worker/query-tasks/{query_task_id}/fail");
1434        let response = self
1435            .request_json(
1436                reqwest::Method::POST,
1437                &path,
1438                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1439                Some(&body),
1440            )
1441            .await;
1442        query_task_response(response)
1443    }
1444
1445    pub async fn heartbeat_worker(
1446        &self,
1447        worker_id: &str,
1448        workflow_available: usize,
1449        activity_available: usize,
1450    ) -> Result<Value> {
1451        let body = json!({
1452            "worker_id": worker_id,
1453            "task_slots": {
1454                "workflow_available": workflow_available,
1455                "activity_available": activity_available
1456            },
1457            "process_metrics": {
1458                "process_id": std::process::id(),
1459                "process_uptime_seconds": 0
1460            }
1461        });
1462
1463        self.request_json(
1464            reqwest::Method::POST,
1465            "/worker/heartbeat",
1466            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1467            Some(&body),
1468        )
1469        .await
1470    }
1471
1472    pub async fn poll_workflow_task(
1473        &self,
1474        worker_id: &str,
1475        task_queue: &str,
1476        timeout: Duration,
1477    ) -> Result<Option<WorkflowTask>> {
1478        Ok(self
1479            .poll_workflow_task_response(worker_id, task_queue, timeout)
1480            .await?
1481            .task)
1482    }
1483
1484    pub async fn poll_workflow_task_response(
1485        &self,
1486        worker_id: &str,
1487        task_queue: &str,
1488        timeout: Duration,
1489    ) -> Result<PollWorkflowTaskResponse> {
1490        let body = json!({
1491            "worker_id": worker_id,
1492            "task_queue": task_queue,
1493            "timeout_seconds": long_poll_timeout_seconds(timeout),
1494        });
1495        let mut data: PollWorkflowTaskResponse = worker_poll_response(
1496            self.request_json_with_timeout(
1497                reqwest::Method::POST,
1498                "/worker/workflow-tasks/poll",
1499                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1500                Some(&body),
1501                timeout + Duration::from_secs(5),
1502            )
1503            .await,
1504        )?;
1505
1506        if let Some(task) = data.task.as_mut() {
1507            self.fetch_remaining_workflow_history(worker_id, task)
1508                .await?;
1509        }
1510
1511        Ok(data)
1512    }
1513
1514    async fn fetch_remaining_workflow_history(
1515        &self,
1516        worker_id: &str,
1517        task: &mut WorkflowTask,
1518    ) -> Result<()> {
1519        let mut next_token = task.next_history_page_token.clone();
1520
1521        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
1522            let lease_owner = task
1523                .lease_owner
1524                .clone()
1525                .unwrap_or_else(|| worker_id.to_string());
1526            let page = self
1527                .workflow_task_history_page(
1528                    &task.task_id,
1529                    &lease_owner,
1530                    task.workflow_task_attempt,
1531                    &token,
1532                )
1533                .await?;
1534
1535            task.append_history_page(page);
1536
1537            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
1538                return Err(Error::Codec(
1539                    "workflow history pagination returned the same page token".to_string(),
1540                ));
1541            }
1542
1543            next_token = task.next_history_page_token.clone();
1544        }
1545
1546        Ok(())
1547    }
1548
1549    async fn workflow_task_history_page(
1550        &self,
1551        task_id: &str,
1552        lease_owner: &str,
1553        workflow_task_attempt: u64,
1554        next_history_page_token: &str,
1555    ) -> Result<WorkflowTaskHistoryPage> {
1556        let body = json!({
1557            "lease_owner": lease_owner,
1558            "workflow_task_attempt": workflow_task_attempt,
1559            "next_history_page_token": next_history_page_token
1560        });
1561        let path = format!("/worker/workflow-tasks/{task_id}/history");
1562
1563        self.request_json(
1564            reqwest::Method::POST,
1565            &path,
1566            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1567            Some(&body),
1568        )
1569        .await
1570    }
1571
1572    pub async fn complete_workflow_task(
1573        &self,
1574        task_id: &str,
1575        lease_owner: &str,
1576        workflow_task_attempt: u64,
1577        commands: Vec<Value>,
1578    ) -> Result<Value> {
1579        let body = json!({
1580            "lease_owner": lease_owner,
1581            "workflow_task_attempt": workflow_task_attempt,
1582            "commands": commands
1583        });
1584        let path = format!("/worker/workflow-tasks/{task_id}/complete");
1585        self.request_json(
1586            reqwest::Method::POST,
1587            &path,
1588            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1589            Some(&body),
1590        )
1591        .await
1592    }
1593
1594    pub async fn fail_workflow_task(
1595        &self,
1596        task_id: &str,
1597        lease_owner: &str,
1598        workflow_task_attempt: u64,
1599        message: impl Into<String>,
1600    ) -> Result<Value> {
1601        self.fail_workflow_task_with_type(
1602            task_id,
1603            lease_owner,
1604            workflow_task_attempt,
1605            message,
1606            "RustWorkflowTaskFailure",
1607        )
1608        .await
1609    }
1610
1611    async fn fail_workflow_task_with_type(
1612        &self,
1613        task_id: &str,
1614        lease_owner: &str,
1615        workflow_task_attempt: u64,
1616        message: impl Into<String>,
1617        failure_type: &str,
1618    ) -> Result<Value> {
1619        let body = json!({
1620            "lease_owner": lease_owner,
1621            "workflow_task_attempt": workflow_task_attempt,
1622            "failure": {
1623                "message": message.into(),
1624                "type": failure_type
1625            }
1626        });
1627        let path = format!("/worker/workflow-tasks/{task_id}/fail");
1628        self.request_json(
1629            reqwest::Method::POST,
1630            &path,
1631            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1632            Some(&body),
1633        )
1634        .await
1635    }
1636
1637    pub async fn poll_activity_task(
1638        &self,
1639        worker_id: &str,
1640        task_queue: &str,
1641        timeout: Duration,
1642    ) -> Result<Option<ActivityTask>> {
1643        Ok(self
1644            .poll_activity_task_response(worker_id, task_queue, timeout)
1645            .await?
1646            .task)
1647    }
1648
1649    /// Poll an activity task while preserving server stop and drain metadata.
1650    pub async fn poll_activity_task_response(
1651        &self,
1652        worker_id: &str,
1653        task_queue: &str,
1654        timeout: Duration,
1655    ) -> Result<PollActivityTaskResponse> {
1656        let body = json!({
1657            "worker_id": worker_id,
1658            "task_queue": task_queue,
1659            "timeout_seconds": long_poll_timeout_seconds(timeout),
1660        });
1661        let data: PollActivityTaskResponse = worker_poll_response(
1662            self.request_json_with_timeout(
1663                reqwest::Method::POST,
1664                "/worker/activity-tasks/poll",
1665                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1666                Some(&body),
1667                timeout + Duration::from_secs(5),
1668            )
1669            .await,
1670        )?;
1671        Ok(data)
1672    }
1673
1674    pub async fn complete_activity_task(
1675        &self,
1676        task_id: &str,
1677        activity_attempt_id: &str,
1678        lease_owner: &str,
1679        result: Value,
1680        codec: &str,
1681    ) -> Result<Value> {
1682        let result = encode_value_envelope(&result, codec)?;
1683        let body = json!({
1684            "activity_attempt_id": activity_attempt_id,
1685            "lease_owner": lease_owner,
1686            "result": result
1687        });
1688        let path = format!("/worker/activity-tasks/{task_id}/complete");
1689        activity_task_response(
1690            self.request_json(
1691                reqwest::Method::POST,
1692                &path,
1693                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1694                Some(&body),
1695            )
1696            .await,
1697            "complete",
1698            task_id,
1699            activity_attempt_id,
1700        )
1701    }
1702
1703    pub async fn fail_activity_task(
1704        &self,
1705        task_id: &str,
1706        activity_attempt_id: &str,
1707        lease_owner: &str,
1708        message: impl Into<String>,
1709        non_retryable: bool,
1710    ) -> Result<Value> {
1711        let body = json!({
1712            "activity_attempt_id": activity_attempt_id,
1713            "lease_owner": lease_owner,
1714            "failure": {
1715                "message": message.into(),
1716                "type": "RustActivityFailure",
1717                "non_retryable": non_retryable
1718            }
1719        });
1720        let path = format!("/worker/activity-tasks/{task_id}/fail");
1721        activity_task_response(
1722            self.request_json(
1723                reqwest::Method::POST,
1724                &path,
1725                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1726                Some(&body),
1727            )
1728            .await,
1729            "fail",
1730            task_id,
1731            activity_attempt_id,
1732        )
1733    }
1734
1735    pub async fn heartbeat_activity_task(
1736        &self,
1737        task_id: &str,
1738        activity_attempt_id: &str,
1739        lease_owner: &str,
1740        details: Value,
1741    ) -> Result<ActivityHeartbeatResponse> {
1742        let body = json!({
1743            "activity_attempt_id": activity_attempt_id,
1744            "lease_owner": lease_owner,
1745            "details": details
1746        });
1747        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
1748        activity_task_response(
1749            self.request_json(
1750                reqwest::Method::POST,
1751                &path,
1752                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1753                Some(&body),
1754            )
1755            .await,
1756            "heartbeat",
1757            task_id,
1758            activity_attempt_id,
1759        )
1760    }
1761
1762    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
1763        &self,
1764        method: reqwest::Method,
1765        path: &str,
1766        protocol: RequestProtocol,
1767        body: Option<&B>,
1768    ) -> Result<T> {
1769        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
1770            .await
1771    }
1772
1773    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
1774        &self,
1775        method: reqwest::Method,
1776        path: &str,
1777        protocol: RequestProtocol,
1778        body: Option<&B>,
1779        timeout: Duration,
1780    ) -> Result<T> {
1781        let mut request = self
1782            .http
1783            .request(method, format!("{}/api{}", self.base_url, path))
1784            .timeout(timeout)
1785            .header(reqwest::header::ACCEPT, "application/json")
1786            .header(reqwest::header::CONTENT_TYPE, "application/json")
1787            .header("X-Namespace", &self.namespace);
1788
1789        match protocol {
1790            RequestProtocol::Worker(version) => {
1791                request = request.header("X-Durable-Workflow-Protocol-Version", version);
1792            }
1793            RequestProtocol::ControlPlane => {
1794                request = request.header(
1795                    "X-Durable-Workflow-Control-Plane-Version",
1796                    CONTROL_PLANE_VERSION,
1797                );
1798            }
1799        }
1800
1801        if let Some(token) = self.auth_token(protocol.is_worker()) {
1802            request = request.bearer_auth(token);
1803        }
1804
1805        if let Some(body) = body {
1806            request = request.json(body);
1807        }
1808
1809        let response = request.send().await?;
1810        let status = response.status();
1811        let bytes = response.bytes().await?;
1812
1813        if !status.is_success() {
1814            let body = String::from_utf8_lossy(&bytes).to_string();
1815            if let Some(protocol) = protocol_failure(status, &body) {
1816                return Err(Error::Protocol(protocol));
1817            }
1818            return Err(Error::Http { status, body });
1819        }
1820
1821        if bytes.is_empty() {
1822            return Ok(serde_json::from_value(Value::Null)?);
1823        }
1824
1825        Ok(serde_json::from_slice(&bytes)?)
1826    }
1827
1828    fn auth_token(&self, worker: bool) -> Option<&str> {
1829        if worker {
1830            self.worker_token
1831                .as_deref()
1832                .or(self.token.as_deref())
1833                .or(self.control_token.as_deref())
1834        } else {
1835            self.control_token
1836                .as_deref()
1837                .or(self.token.as_deref())
1838                .or(self.worker_token.as_deref())
1839        }
1840    }
1841}
1842
1843fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
1844    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1845    let reason = body
1846        .get("reason")
1847        .and_then(Value::as_str)
1848        .unwrap_or("query_rejected")
1849        .to_string();
1850    let message = body
1851        .get("message")
1852        .or_else(|| body.get("error"))
1853        .and_then(Value::as_str)
1854        .unwrap_or("workflow query was rejected")
1855        .to_string();
1856
1857    QueryFailure {
1858        status: status.as_u16(),
1859        reason,
1860        message,
1861        body,
1862    }
1863}
1864
1865fn workflow_command_result(
1866    command: WorkflowCommandKind,
1867    data: Value,
1868    workflow_id: &str,
1869    run_id: Option<&str>,
1870) -> WorkflowCommandResult {
1871    WorkflowCommandResult {
1872        command,
1873        workflow_id: data
1874            .get("workflow_id")
1875            .and_then(Value::as_str)
1876            .unwrap_or(workflow_id)
1877            .to_string(),
1878        run_id: data
1879            .get("run_id")
1880            .and_then(Value::as_str)
1881            .or(run_id)
1882            .map(str::to_string),
1883        outcome: data
1884            .get("outcome")
1885            .and_then(Value::as_str)
1886            .map(str::to_string),
1887        reason: data
1888            .get("reason")
1889            .and_then(Value::as_str)
1890            .map(str::to_string),
1891        command_status: data
1892            .get("command_status")
1893            .and_then(Value::as_str)
1894            .map(str::to_string),
1895        raw: data,
1896    }
1897}
1898
1899fn workflow_command_rejection(
1900    command: WorkflowCommandKind,
1901    status: reqwest::StatusCode,
1902    raw_body: String,
1903    workflow_id: &str,
1904    run_id: Option<&str>,
1905) -> WorkflowCommandRejection {
1906    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1907    WorkflowCommandRejection {
1908        command,
1909        status: status.as_u16(),
1910        reason: body
1911            .get("reason")
1912            .and_then(Value::as_str)
1913            .unwrap_or("workflow_command_rejected")
1914            .to_string(),
1915        message: body
1916            .get("message")
1917            .or_else(|| body.get("error"))
1918            .and_then(Value::as_str)
1919            .unwrap_or("workflow lifecycle command was rejected")
1920            .to_string(),
1921        workflow_id: body
1922            .get("workflow_id")
1923            .and_then(Value::as_str)
1924            .unwrap_or(workflow_id)
1925            .to_string(),
1926        run_id: body
1927            .get("run_id")
1928            .and_then(Value::as_str)
1929            .or(run_id)
1930            .map(str::to_string),
1931        target_scope: body
1932            .get("target_scope")
1933            .and_then(Value::as_str)
1934            .map(str::to_string),
1935        body,
1936    }
1937}
1938
1939fn query_task_response(response: Result<Value>) -> Result<Value> {
1940    match response {
1941        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
1942        response => response,
1943    }
1944}
1945
1946fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
1947    match response {
1948        Err(Error::Http { status, body })
1949            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
1950        {
1951            Ok(serde_json::from_str(&body)?)
1952        }
1953        response => response,
1954    }
1955}
1956
1957fn worker_poll_body_is_stop(body: &str) -> bool {
1958    serde_json::from_str::<Value>(body)
1959        .ok()
1960        .is_some_and(|body| {
1961            worker_poll_is_stop(
1962                body.get("poll_status").and_then(Value::as_str),
1963                body.get("reason").and_then(Value::as_str),
1964            )
1965        })
1966}
1967
1968fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
1969    matches!(poll_status, Some("draining" | "stopped"))
1970        || matches!(reason, Some("worker_draining" | "worker_stopped"))
1971}
1972
1973fn query_task_rejection_is_final(error: &Error) -> bool {
1974    matches!(
1975        error,
1976        Error::QueryFailed(failure)
1977            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
1978    )
1979}
1980
1981fn activity_task_response<T>(
1982    response: Result<T>,
1983    operation: &str,
1984    task_id: &str,
1985    activity_attempt_id: &str,
1986) -> Result<T> {
1987    match response {
1988        Err(Error::Http { status, body }) => {
1989            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
1990            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
1991                operation: operation.to_string(),
1992                status: status.as_u16(),
1993                reason: body
1994                    .get("reason")
1995                    .and_then(Value::as_str)
1996                    .unwrap_or("activity_task_rejected")
1997                    .to_string(),
1998                task_id: body
1999                    .get("task_id")
2000                    .and_then(Value::as_str)
2001                    .unwrap_or(task_id)
2002                    .to_string(),
2003                activity_attempt_id: body
2004                    .get("activity_attempt_id")
2005                    .and_then(Value::as_str)
2006                    .unwrap_or(activity_attempt_id)
2007                    .to_string(),
2008                cancel_requested: body
2009                    .get("cancel_requested")
2010                    .and_then(Value::as_bool)
2011                    .unwrap_or(false),
2012                can_continue: body.get("can_continue").and_then(Value::as_bool),
2013                run_closed_reason: body
2014                    .get("run_closed_reason")
2015                    .and_then(Value::as_str)
2016                    .map(str::to_string),
2017                body,
2018            }))
2019        }
2020        response => response,
2021    }
2022}
2023
2024fn activity_task_rejection_is_final(error: &Error) -> bool {
2025    matches!(
2026        error,
2027        Error::ActivityTaskRejected(rejection)
2028            if matches!(
2029                rejection.reason.as_str(),
2030                "run_cancelled"
2031                    | "run_terminated"
2032                    | "attempt_closed"
2033                    | "stale_attempt"
2034                    | "activity_cancelled"
2035                    | "task_cancelled"
2036                    | "run_closed"
2037                    | "activity_not_running"
2038                    | "attempt_not_found"
2039            )
2040    )
2041}
2042
2043fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2044    let body: Value = serde_json::from_str(raw_body).ok()?;
2045    let reason = body.get("reason")?.as_str()?;
2046    if !matches!(
2047        reason,
2048        "missing_protocol_version"
2049            | "unsupported_protocol_version"
2050            | "missing_control_plane_version"
2051            | "unsupported_control_plane_version"
2052    ) {
2053        return None;
2054    }
2055
2056    Some(ProtocolFailure {
2057        status: status.as_u16(),
2058        reason: reason.to_string(),
2059        message: body
2060            .get("message")
2061            .or_else(|| body.get("error"))
2062            .and_then(Value::as_str)
2063            .unwrap_or("protocol version rejected")
2064            .to_string(),
2065        supported_version: body
2066            .get("supported_version")
2067            .and_then(Value::as_str)
2068            .map(str::to_string),
2069        requested_version: body
2070            .get("requested_version")
2071            .and_then(Value::as_str)
2072            .map(str::to_string),
2073        body,
2074    })
2075}
2076
2077fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2078    timeout
2079        .as_secs()
2080        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2081        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2082}
2083
2084fn worker_operation_is_retryable(error: &Error) -> bool {
2085    match error {
2086        Error::Transport(error) => {
2087            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2088        }
2089        Error::Http { status, .. } => {
2090            matches!(
2091                *status,
2092                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2093            ) || status.is_server_error()
2094        }
2095        _ => false,
2096    }
2097}
2098
2099fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
2100    let exponent = retry.saturating_sub(1).min(31) as u32;
2101    policy
2102        .initial_backoff
2103        .saturating_mul(1_u32 << exponent)
2104        .min(policy.max_backoff)
2105}
2106
2107#[derive(Debug)]
2108pub struct ClientBuilder {
2109    base_url: String,
2110    token: Option<String>,
2111    control_token: Option<String>,
2112    worker_token: Option<String>,
2113    namespace: String,
2114    timeout: Duration,
2115}
2116
2117impl ClientBuilder {
2118    pub fn token(mut self, token: Option<String>) -> Self {
2119        self.token = token;
2120        self
2121    }
2122
2123    pub fn control_token(mut self, token: Option<String>) -> Self {
2124        self.control_token = token;
2125        self
2126    }
2127
2128    pub fn worker_token(mut self, token: Option<String>) -> Self {
2129        self.worker_token = token;
2130        self
2131    }
2132
2133    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
2134        self.namespace = namespace.into();
2135        self
2136    }
2137
2138    pub fn timeout(mut self, timeout: Duration) -> Self {
2139        self.timeout = timeout;
2140        self
2141    }
2142
2143    pub fn build(self) -> Result<Client> {
2144        Ok(Client {
2145            http: reqwest::Client::builder().timeout(self.timeout).build()?,
2146            base_url: self.base_url.trim_end_matches('/').to_string(),
2147            token: self.token,
2148            control_token: self.control_token,
2149            worker_token: self.worker_token,
2150            namespace: self.namespace,
2151        })
2152    }
2153}
2154
2155#[derive(Clone, Debug)]
2156pub struct WorkflowHandle {
2157    client: Client,
2158    pub workflow_id: String,
2159    pub run_id: Option<String>,
2160    pub workflow_type: String,
2161}
2162
2163impl WorkflowHandle {
2164    pub async fn describe(&self) -> Result<WorkflowDescription> {
2165        self.client.describe_workflow(&self.workflow_id).await
2166    }
2167
2168    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
2169        self.client
2170            .signal_workflow(&self.workflow_id, signal_name, input)
2171            .await
2172    }
2173
2174    /// Request cooperative cancellation of whichever run is current.
2175    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
2176        self.client
2177            .cancel_workflow(&self.workflow_id, options)
2178            .await
2179    }
2180
2181    /// Request cancellation only if this handle's selected run is still current.
2182    pub async fn cancel_selected_run(
2183        &self,
2184        options: WorkflowCommandOptions,
2185    ) -> Result<WorkflowCommandResult> {
2186        let run_id = self.run_id.as_deref().ok_or_else(|| {
2187            Error::Codec("run_id is required for selected-run cancellation".to_string())
2188        })?;
2189        self.client
2190            .cancel_workflow_run(&self.workflow_id, run_id, options)
2191            .await
2192    }
2193
2194    /// Forcefully terminate whichever run is current.
2195    pub async fn terminate(
2196        &self,
2197        options: WorkflowCommandOptions,
2198    ) -> Result<WorkflowCommandResult> {
2199        self.client
2200            .terminate_workflow(&self.workflow_id, options)
2201            .await
2202    }
2203
2204    /// Terminate only if this handle's selected run is still current.
2205    pub async fn terminate_selected_run(
2206        &self,
2207        options: WorkflowCommandOptions,
2208    ) -> Result<WorkflowCommandResult> {
2209        let run_id = self.run_id.as_deref().ok_or_else(|| {
2210            Error::Codec("run_id is required for selected-run termination".to_string())
2211        })?;
2212        self.client
2213            .terminate_workflow_run(&self.workflow_id, run_id, options)
2214            .await
2215    }
2216
2217    /// Execute a named, read-only query against this workflow.
2218    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
2219        self.client
2220            .query_workflow(&self.workflow_id, query_name, input)
2221            .await
2222    }
2223
2224    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
2225        let started = Instant::now();
2226
2227        loop {
2228            let description = match self.run_id.as_deref() {
2229                Some(run_id) => {
2230                    self.client
2231                        .describe_workflow_run(&self.workflow_id, run_id)
2232                        .await?
2233                }
2234                None => self.describe().await?,
2235            };
2236            if description.is_completed() {
2237                return Ok(description.output.unwrap_or(Value::Null));
2238            }
2239
2240            if description.is_terminal() {
2241                let outcome = workflow_terminal_outcome(
2242                    &description,
2243                    &self.workflow_id,
2244                    self.run_id.as_deref(),
2245                );
2246                return Err(match outcome.kind {
2247                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
2248                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
2249                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
2250                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
2251                });
2252            }
2253
2254            if started.elapsed() >= options.timeout {
2255                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
2256                    kind: WorkflowTerminalKind::TimedOut,
2257                    workflow_id: description
2258                        .workflow_id
2259                        .clone()
2260                        .unwrap_or_else(|| self.workflow_id.clone()),
2261                    run_id: description.run_id.clone().or_else(|| self.run_id.clone()),
2262                    reason: "result_wait_timeout".to_string(),
2263                    failure_category: Some("client_timeout".to_string()),
2264                    failure_id: None,
2265                    exception_type: None,
2266                    exception_class: None,
2267                    non_retryable: None,
2268                    message: Some(format!(
2269                        "workflow result was not terminal within {:?}",
2270                        options.timeout
2271                    )),
2272                    exception: None,
2273                    raw: description.raw_value(),
2274                }));
2275            }
2276
2277            tokio::time::sleep(options.poll_interval).await;
2278        }
2279    }
2280}
2281
2282#[derive(Clone, Copy, Debug)]
2283pub struct WorkflowResultOptions {
2284    pub poll_interval: Duration,
2285    pub timeout: Duration,
2286}
2287
2288impl Default for WorkflowResultOptions {
2289    fn default() -> Self {
2290        Self {
2291            poll_interval: Duration::from_millis(500),
2292            timeout: Duration::from_secs(30),
2293        }
2294    }
2295}
2296
2297#[derive(Clone, Debug, Deserialize)]
2298pub struct WorkflowDescription {
2299    pub workflow_id: Option<String>,
2300    pub run_id: Option<String>,
2301    pub workflow_type: Option<String>,
2302    pub status: Option<String>,
2303    #[serde(default)]
2304    pub closed_reason: Option<String>,
2305    #[serde(default)]
2306    pub error: Option<String>,
2307    #[serde(default)]
2308    pub failure: Option<Value>,
2309    #[serde(default)]
2310    pub exception: Option<Value>,
2311    #[serde(default)]
2312    pub failures: Vec<Value>,
2313    #[serde(default)]
2314    pub output: Option<Value>,
2315    #[serde(default)]
2316    pub output_envelope: Option<Value>,
2317    #[serde(flatten)]
2318    pub raw: HashMap<String, Value>,
2319}
2320
2321impl WorkflowDescription {
2322    pub fn is_completed(&self) -> bool {
2323        matches!(self.status.as_deref(), Some("completed" | "Completed"))
2324    }
2325
2326    pub fn is_terminal(&self) -> bool {
2327        matches!(
2328            self.status.as_deref(),
2329            Some(
2330                "completed"
2331                    | "Completed"
2332                    | "failed"
2333                    | "Failed"
2334                    | "cancelled"
2335                    | "Cancelled"
2336                    | "terminated"
2337                    | "Terminated"
2338                    | "timed_out"
2339                    | "TimedOut",
2340            )
2341        )
2342    }
2343
2344    fn decode_payloads(&mut self) -> Result<()> {
2345        if let Some(envelope) = &self.output_envelope {
2346            self.output = Some(decode_wire_value(envelope, DEFAULT_CODEC)?);
2347        }
2348
2349        Ok(())
2350    }
2351
2352    fn raw_value(&self) -> Value {
2353        let mut data = self.raw.clone();
2354        data.insert(
2355            "workflow_id".to_string(),
2356            self.workflow_id
2357                .clone()
2358                .map(Value::String)
2359                .unwrap_or(Value::Null),
2360        );
2361        data.insert(
2362            "run_id".to_string(),
2363            self.run_id
2364                .clone()
2365                .map(Value::String)
2366                .unwrap_or(Value::Null),
2367        );
2368        data.insert(
2369            "workflow_type".to_string(),
2370            self.workflow_type
2371                .clone()
2372                .map(Value::String)
2373                .unwrap_or(Value::Null),
2374        );
2375        data.insert(
2376            "status".to_string(),
2377            self.status
2378                .clone()
2379                .map(Value::String)
2380                .unwrap_or(Value::Null),
2381        );
2382        data.insert(
2383            "closed_reason".to_string(),
2384            self.closed_reason
2385                .clone()
2386                .map(Value::String)
2387                .unwrap_or(Value::Null),
2388        );
2389        if let Some(failure) = &self.failure {
2390            data.insert("failure".to_string(), failure.clone());
2391        }
2392        if let Some(exception) = &self.exception {
2393            data.insert("exception".to_string(), exception.clone());
2394        }
2395        Value::Object(data.into_iter().collect())
2396    }
2397}
2398
2399fn workflow_terminal_outcome(
2400    description: &WorkflowDescription,
2401    workflow_id: &str,
2402    run_id: Option<&str>,
2403) -> WorkflowTerminalOutcome {
2404    let terminal_kind = description
2405        .closed_reason
2406        .as_deref()
2407        .or(description.status.as_deref())
2408        .unwrap_or("failed")
2409        .to_ascii_lowercase();
2410    let kind = match terminal_kind.as_str() {
2411        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
2412        "terminated" => WorkflowTerminalKind::Terminated,
2413        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
2414        _ => WorkflowTerminalKind::Failed,
2415    };
2416    let default_reason = match kind {
2417        WorkflowTerminalKind::Failed => "workflow_failed",
2418        WorkflowTerminalKind::Cancelled => "cancelled",
2419        WorkflowTerminalKind::Terminated => "terminated",
2420        WorkflowTerminalKind::TimedOut => "timed_out",
2421    };
2422    let failure = description
2423        .failure
2424        .as_ref()
2425        .filter(|value| value.is_object());
2426    let nested_failure = failure
2427        .and_then(|value| value.get("failures"))
2428        .and_then(Value::as_array)
2429        .and_then(|failures| failures.last())
2430        .or_else(|| description.failures.last());
2431    let exception = description
2432        .exception
2433        .clone()
2434        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
2435        .or_else(|| {
2436            nested_failure
2437                .and_then(|value| value.get("exception_payload"))
2438                .cloned()
2439        });
2440    let string_field = |name: &str| {
2441        failure
2442            .and_then(|value| value.get(name))
2443            .and_then(Value::as_str)
2444            .or_else(|| {
2445                nested_failure
2446                    .and_then(|value| value.get(name))
2447                    .and_then(Value::as_str)
2448            })
2449            .map(str::to_string)
2450    };
2451    let exception_field = |name: &str| {
2452        exception
2453            .as_ref()
2454            .and_then(|value| value.get(name))
2455            .and_then(Value::as_str)
2456            .map(str::to_string)
2457    };
2458    let message = description
2459        .error
2460        .clone()
2461        .or_else(|| string_field("message"))
2462        .or_else(|| exception_field("message"));
2463    let reason = description
2464        .raw
2465        .get("reason")
2466        .and_then(Value::as_str)
2467        .map(str::to_string)
2468        .or_else(|| {
2469            failure
2470                .and_then(|value| value.get("reason"))
2471                .and_then(Value::as_str)
2472                .map(str::to_string)
2473        })
2474        .or_else(|| description.closed_reason.clone())
2475        .unwrap_or_else(|| default_reason.to_string());
2476    let failure_id = string_field("failure_id").or_else(|| {
2477        nested_failure
2478            .and_then(|value| value.get("id"))
2479            .and_then(Value::as_str)
2480            .map(str::to_string)
2481    });
2482
2483    WorkflowTerminalOutcome {
2484        kind,
2485        workflow_id: description
2486            .workflow_id
2487            .clone()
2488            .unwrap_or_else(|| workflow_id.to_string()),
2489        run_id: description
2490            .run_id
2491            .clone()
2492            .or_else(|| run_id.map(str::to_string)),
2493        reason,
2494        failure_category: string_field("failure_category")
2495            .or_else(|| Some(default_reason.to_string())),
2496        failure_id,
2497        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
2498        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
2499        non_retryable: failure
2500            .and_then(|value| value.get("non_retryable"))
2501            .and_then(Value::as_bool)
2502            .or_else(|| {
2503                nested_failure
2504                    .and_then(|value| value.get("non_retryable"))
2505                    .and_then(Value::as_bool)
2506            }),
2507        message,
2508        exception,
2509        raw: description.raw_value(),
2510    }
2511}
2512
2513#[derive(Clone, Debug, Deserialize)]
2514pub struct RegisterWorkerResponse {
2515    pub worker_id: String,
2516    pub registered: bool,
2517    #[serde(default)]
2518    pub heartbeat_interval_seconds: Option<u64>,
2519    #[serde(default)]
2520    pub protocol_version: Option<String>,
2521    #[serde(default)]
2522    pub server_capabilities: Option<Value>,
2523}
2524
2525#[derive(Clone, Debug, Deserialize)]
2526pub struct PollWorkflowTaskResponse {
2527    #[serde(default)]
2528    pub task: Option<WorkflowTask>,
2529    #[serde(default)]
2530    pub poll_status: Option<String>,
2531    #[serde(default)]
2532    pub reason: Option<String>,
2533    #[serde(default)]
2534    pub protocol_version: Option<String>,
2535    #[serde(default)]
2536    pub server_capabilities: Option<Value>,
2537}
2538
2539impl PollWorkflowTaskResponse {
2540    /// Classify this response without parsing server display text.
2541    pub fn outcome(&self) -> WorkerPollOutcome {
2542        worker_poll_outcome(
2543            self.task.is_some(),
2544            self.poll_status.as_deref(),
2545            self.reason.as_deref(),
2546        )
2547    }
2548}
2549
2550#[derive(Clone, Debug, Deserialize)]
2551pub struct PollActivityTaskResponse {
2552    #[serde(default)]
2553    pub task: Option<ActivityTask>,
2554    #[serde(default)]
2555    pub poll_status: Option<String>,
2556    #[serde(default)]
2557    pub reason: Option<String>,
2558}
2559
2560impl PollActivityTaskResponse {
2561    /// Classify this response without parsing server display text.
2562    pub fn outcome(&self) -> WorkerPollOutcome {
2563        worker_poll_outcome(
2564            self.task.is_some(),
2565            self.poll_status.as_deref(),
2566            self.reason.as_deref(),
2567        )
2568    }
2569}
2570
2571#[derive(Clone, Debug, Deserialize)]
2572pub struct PollQueryTaskResponse {
2573    #[serde(default)]
2574    pub task: Option<QueryTask>,
2575    #[serde(default)]
2576    pub poll_status: Option<String>,
2577    #[serde(default)]
2578    pub reason: Option<String>,
2579}
2580
2581impl PollQueryTaskResponse {
2582    /// Classify this response without parsing server display text.
2583    pub fn outcome(&self) -> WorkerPollOutcome {
2584        worker_poll_outcome(
2585            self.task.is_some(),
2586            self.poll_status.as_deref(),
2587            self.reason.as_deref(),
2588        )
2589    }
2590}
2591
2592/// Stable classification for worker poll responses.
2593#[derive(Clone, Debug, PartialEq, Eq)]
2594pub enum WorkerPollOutcome {
2595    /// A task was leased and is available on the response.
2596    Task,
2597    /// No task was leased, but the worker should continue polling.
2598    Idle {
2599        poll_status: Option<String>,
2600        reason: Option<String>,
2601    },
2602    /// The server asked this worker to stop claiming new work.
2603    Stop {
2604        poll_status: Option<String>,
2605        reason: Option<String>,
2606    },
2607}
2608
2609impl WorkerPollOutcome {
2610    pub fn should_stop(&self) -> bool {
2611        matches!(self, Self::Stop { .. })
2612    }
2613}
2614
2615fn worker_poll_outcome(
2616    has_task: bool,
2617    poll_status: Option<&str>,
2618    reason: Option<&str>,
2619) -> WorkerPollOutcome {
2620    if worker_poll_is_stop(poll_status, reason) {
2621        return WorkerPollOutcome::Stop {
2622            poll_status: poll_status.map(str::to_string),
2623            reason: reason.map(str::to_string),
2624        };
2625    }
2626
2627    if has_task {
2628        WorkerPollOutcome::Task
2629    } else {
2630        WorkerPollOutcome::Idle {
2631            poll_status: poll_status.map(str::to_string),
2632            reason: reason.map(str::to_string),
2633        }
2634    }
2635}
2636
2637/// An ephemeral server-routed query task.
2638#[derive(Clone, Debug, Deserialize)]
2639pub struct QueryTask {
2640    pub query_task_id: String,
2641    #[serde(default = "default_workflow_task_attempt")]
2642    pub query_task_attempt: u64,
2643    #[serde(default)]
2644    pub lease_owner: Option<String>,
2645    #[serde(default)]
2646    pub workflow_id: Option<String>,
2647    #[serde(default)]
2648    pub run_id: Option<String>,
2649    pub workflow_type: String,
2650    pub query_name: String,
2651    #[serde(default = "default_payload_codec")]
2652    pub payload_codec: String,
2653    #[serde(default)]
2654    pub workflow_arguments: Option<Value>,
2655    #[serde(default)]
2656    pub query_arguments: Option<Value>,
2657    #[serde(default)]
2658    pub history_events: Vec<HistoryEvent>,
2659    #[serde(default)]
2660    pub history_export: Option<Value>,
2661    #[serde(default)]
2662    pub run_status: Option<String>,
2663}
2664
2665#[derive(Clone, Debug, Deserialize)]
2666pub struct WorkflowTask {
2667    pub task_id: String,
2668    #[serde(default)]
2669    pub workflow_id: Option<String>,
2670    #[serde(default)]
2671    pub run_id: Option<String>,
2672    pub workflow_type: String,
2673    #[serde(default = "default_payload_codec")]
2674    pub payload_codec: String,
2675    #[serde(default)]
2676    pub arguments: Option<Value>,
2677    #[serde(default)]
2678    pub history_events: Vec<HistoryEvent>,
2679    #[serde(default)]
2680    pub total_history_events: Option<u64>,
2681    #[serde(default)]
2682    pub next_history_page_token: Option<String>,
2683    #[serde(default = "default_workflow_task_attempt")]
2684    pub workflow_task_attempt: u64,
2685    #[serde(default)]
2686    pub workflow_signal_id: Option<String>,
2687    #[serde(default)]
2688    pub signal_name: Option<String>,
2689    #[serde(default)]
2690    pub signal_arguments: Option<Value>,
2691    #[serde(default)]
2692    pub lease_owner: Option<String>,
2693}
2694
2695impl WorkflowTask {
2696    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
2697        self.history_events.extend(page.history_events);
2698
2699        if page.total_history_events.is_some() {
2700            self.total_history_events = page.total_history_events;
2701        }
2702
2703        self.next_history_page_token = page
2704            .next_history_page_token
2705            .filter(|token| !token.is_empty());
2706    }
2707}
2708
2709#[derive(Clone, Debug, Deserialize)]
2710struct WorkflowTaskHistoryPage {
2711    #[serde(default)]
2712    history_events: Vec<HistoryEvent>,
2713    #[serde(default)]
2714    total_history_events: Option<u64>,
2715    #[serde(default)]
2716    next_history_page_token: Option<String>,
2717}
2718
2719#[derive(Clone, Debug, Deserialize)]
2720pub struct ActivityTask {
2721    pub task_id: String,
2722    #[serde(default)]
2723    pub activity_attempt_id: Option<String>,
2724    #[serde(default)]
2725    pub attempt_id: Option<String>,
2726    pub activity_type: String,
2727    #[serde(default = "default_payload_codec")]
2728    pub payload_codec: String,
2729    #[serde(default)]
2730    pub arguments: Option<Value>,
2731    #[serde(default = "default_attempt_number")]
2732    pub attempt_number: u64,
2733    #[serde(default)]
2734    pub lease_owner: Option<String>,
2735}
2736
2737#[derive(Clone, Debug, Deserialize)]
2738pub struct HistoryEvent {
2739    #[serde(alias = "type")]
2740    pub event_type: String,
2741    #[serde(default)]
2742    pub payload: Value,
2743    #[serde(flatten)]
2744    pub raw: HashMap<String, Value>,
2745}
2746
2747/// One decoded signal in the committed workflow-history snapshot.
2748#[derive(Clone, Debug, PartialEq)]
2749pub struct QuerySignal {
2750    pub id: Option<String>,
2751    pub name: String,
2752    pub arguments: Vec<Value>,
2753    pub workflow_sequence: Option<u64>,
2754}
2755
2756/// Immutable state supplied to a registered query handler.
2757///
2758/// This context intentionally exposes no activity, signal-wait, or command
2759/// APIs. Query handlers inspect committed history and return a value; query
2760/// completion does not append an event or advance deterministic execution.
2761#[derive(Clone, Debug)]
2762pub struct QueryContext {
2763    pub workflow_id: Option<String>,
2764    pub run_id: Option<String>,
2765    pub workflow_type: String,
2766    pub run_status: Option<String>,
2767    workflow_input: Value,
2768    history_events: Arc<Vec<HistoryEvent>>,
2769    signal_events: Arc<Vec<QuerySignal>>,
2770}
2771
2772impl QueryContext {
2773    /// The normalized argument list used to start the workflow.
2774    pub fn workflow_input(&self) -> &Value {
2775        &self.workflow_input
2776    }
2777
2778    /// The immutable committed history used for this query snapshot.
2779    pub fn history_events(&self) -> &[HistoryEvent] {
2780        self.history_events.as_slice()
2781    }
2782
2783    /// All decoded signals in committed workflow order.
2784    pub fn signal_events(&self) -> &[QuerySignal] {
2785        self.signal_events.as_slice()
2786    }
2787
2788    /// Decoded argument lists for each committed signal with `signal_name`.
2789    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
2790        self.signal_events
2791            .iter()
2792            .filter(|signal| signal.name == signal_name)
2793            .map(|signal| signal.arguments.clone())
2794            .collect()
2795    }
2796}
2797
2798#[derive(Clone, Debug, Deserialize)]
2799pub struct ActivityHeartbeatResponse {
2800    #[serde(default)]
2801    pub cancel_requested: bool,
2802    #[serde(default)]
2803    pub heartbeat_recorded: bool,
2804    #[serde(default)]
2805    pub can_continue: Option<bool>,
2806    #[serde(default)]
2807    pub reason: Option<String>,
2808    #[serde(default)]
2809    pub run_closed_reason: Option<String>,
2810    #[serde(default)]
2811    pub run_closed_at: Option<String>,
2812    #[serde(default)]
2813    pub lease_expires_at: Option<String>,
2814    #[serde(default)]
2815    pub last_heartbeat_at: Option<String>,
2816}
2817
2818impl ActivityHeartbeatResponse {
2819    /// Whether the activity should stop instead of attempting completion.
2820    pub fn should_stop(&self) -> bool {
2821        self.cancel_requested || self.can_continue == Some(false)
2822    }
2823}
2824
2825fn default_payload_codec() -> String {
2826    DEFAULT_CODEC.to_string()
2827}
2828
2829fn default_workflow_task_attempt() -> u64 {
2830    1
2831}
2832
2833fn default_attempt_number() -> u64 {
2834    1
2835}
2836
2837type WorkflowFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2838type WorkflowHandler = Arc<dyn Fn(WorkflowContext, Value) -> WorkflowFuture + Send + Sync>;
2839type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
2840type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
2841type ReplayedWorkflowHandler =
2842    Arc<dyn Fn(WorkflowContext, Value) -> ReplayedWorkflowInvocation + Send + Sync>;
2843type ActivityFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2844type ActivityHandler = Arc<dyn Fn(ActivityContext, Value) -> ActivityFuture + Send + Sync>;
2845type QueryFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2846type QueryHandler = Arc<dyn Fn(QueryContext, Value) -> QueryFuture + Send + Sync>;
2847type ReplayedQueryHandler = Arc<
2848    dyn Fn(QueryContext, ErasedWorkflowState, Value) -> std::result::Result<QueryFuture, String>
2849        + Send
2850        + Sync,
2851>;
2852type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
2853
2854struct ReplayedWorkflowInvocation {
2855    future: WorkflowFuture,
2856    snapshot: WorkflowStateSnapshot,
2857}
2858
2859#[derive(Clone)]
2860struct RegisteredWorkflow {
2861    execute: WorkflowHandler,
2862    replay: Option<ReplayedWorkflowHandler>,
2863    state_type: Option<TypeId>,
2864}
2865
2866#[derive(Clone)]
2867enum RegisteredQuery {
2868    Snapshot(QueryHandler),
2869    Replayed {
2870        state_type: TypeId,
2871        handler: ReplayedQueryHandler,
2872    },
2873}
2874
2875#[derive(Clone, Debug)]
2876pub struct WorkerHeartbeatObservation {
2877    pub worker_id: String,
2878    pub task_queue: String,
2879    pub acknowledged_at_unix_millis: u64,
2880    pub acknowledgement: Value,
2881}
2882
2883/// Bounded retry policy for worker poll acquisition and worker heartbeats.
2884///
2885/// Expected empty long polls are normal successful responses. Transport
2886/// failures, HTTP 408/429 responses, and server errors are retried with capped
2887/// exponential backoff. Authentication, protocol, codec, and handler failures
2888/// are never retried by the worker.
2889#[derive(Clone, Copy, Debug)]
2890pub struct WorkerRetryPolicy {
2891    /// Number of retries after the initial request fails.
2892    pub max_retries: usize,
2893    /// Delay before the first retry.
2894    pub initial_backoff: Duration,
2895    /// Maximum delay between retries.
2896    pub max_backoff: Duration,
2897}
2898
2899impl Default for WorkerRetryPolicy {
2900    fn default() -> Self {
2901        Self {
2902            max_retries: 5,
2903            initial_backoff: Duration::from_millis(100),
2904            max_backoff: Duration::from_secs(5),
2905        }
2906    }
2907}
2908
2909#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2910enum ManagedPollOutcome {
2911    Idle,
2912    Handled,
2913    Stop,
2914}
2915
2916#[derive(Clone)]
2917pub struct Worker {
2918    client: Client,
2919    worker_id: String,
2920    task_queue: String,
2921    workflows: HashMap<String, RegisteredWorkflow>,
2922    activities: HashMap<String, ActivityHandler>,
2923    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
2924    max_concurrent_workflow_tasks: usize,
2925    max_concurrent_activity_tasks: usize,
2926    poll_timeout: Duration,
2927    heartbeat_interval: Duration,
2928    retry_policy: WorkerRetryPolicy,
2929    heartbeat_observer: Option<WorkerHeartbeatObserver>,
2930}
2931
2932impl Worker {
2933    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
2934        Self {
2935            client,
2936            worker_id: default_worker_id(),
2937            task_queue: task_queue.into(),
2938            workflows: HashMap::new(),
2939            activities: HashMap::new(),
2940            queries: HashMap::new(),
2941            max_concurrent_workflow_tasks: 10,
2942            max_concurrent_activity_tasks: 10,
2943            poll_timeout: Duration::from_secs(30),
2944            heartbeat_interval: Duration::from_secs(60),
2945            retry_policy: WorkerRetryPolicy::default(),
2946            heartbeat_observer: None,
2947        }
2948    }
2949
2950    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
2951        self.worker_id = worker_id.into();
2952        self
2953    }
2954
2955    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
2956        self.poll_timeout = timeout;
2957        self
2958    }
2959
2960    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
2961        self.heartbeat_interval = interval;
2962        self
2963    }
2964
2965    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
2966    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
2967        self.retry_policy = policy;
2968        self
2969    }
2970
2971    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
2972    where
2973        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
2974    {
2975        self.heartbeat_observer = Some(Arc::new(observer));
2976        self
2977    }
2978
2979    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
2980        self.max_concurrent_workflow_tasks = count.max(1);
2981        self
2982    }
2983
2984    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
2985        self.max_concurrent_activity_tasks = count.max(1);
2986        self
2987    }
2988
2989    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
2990    where
2991        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
2992        Fut: Future<Output = Result<Value>> + Send + 'static,
2993    {
2994        self.workflows.insert(
2995            workflow_type.into(),
2996            RegisteredWorkflow {
2997                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
2998                replay: None,
2999                state_type: None,
3000            },
3001        );
3002    }
3003
3004    /// Register a workflow whose typed instance state can be reconstructed for queries.
3005    ///
3006    /// `state_factory` creates a fresh instance for every normal workflow task and
3007    /// query replay. The workflow handler is the single source of truth for state
3008    /// transitions: it updates [`WorkflowInstance`] after activities and signals
3009    /// resolve. Query replay runs this same handler over committed history and
3010    /// discards any commands it would emit.
3011    pub fn register_replayed_workflow<S, Factory, F, Fut>(
3012        &mut self,
3013        workflow_type: impl Into<String>,
3014        state_factory: Factory,
3015        handler: F,
3016    ) where
3017        S: Clone + Send + Sync + 'static,
3018        Factory: Fn() -> S + Send + Sync + 'static,
3019        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
3020        Fut: Future<Output = Result<Value>> + Send + 'static,
3021    {
3022        let state_factory = Arc::new(state_factory);
3023        let handler = Arc::new(handler);
3024
3025        let execute_factory = Arc::clone(&state_factory);
3026        let execute_handler = Arc::clone(&handler);
3027        let execute = Arc::new(move |ctx: WorkflowContext, input: Value| {
3028            let state = WorkflowInstance::new(execute_factory());
3029            let future = execute_handler(ctx, input, state);
3030            Box::pin(future) as WorkflowFuture
3031        });
3032
3033        let replay = Arc::new(move |ctx: WorkflowContext, input: Value| {
3034            let state = WorkflowInstance::new(state_factory());
3035            let snapshot_state = state.clone();
3036            let snapshot: WorkflowStateSnapshot =
3037                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
3038            let future = handler(ctx, input, state);
3039            ReplayedWorkflowInvocation {
3040                future: Box::pin(future),
3041                snapshot,
3042            }
3043        });
3044
3045        self.workflows.insert(
3046            workflow_type.into(),
3047            RegisteredWorkflow {
3048                execute,
3049                replay: Some(replay),
3050                state_type: Some(TypeId::of::<S>()),
3051            },
3052        );
3053    }
3054
3055    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
3056    where
3057        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
3058        Fut: Future<Output = Result<Value>> + Send + 'static,
3059    {
3060        self.activities.insert(
3061            activity_type.into(),
3062            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
3063        );
3064    }
3065
3066    /// Register a named, read-only query handler for a workflow type.
3067    ///
3068    /// The workflow type must also be registered with [`Worker::register_workflow`]
3069    /// before the worker runs. The handler receives only an immutable committed
3070    /// state snapshot and normalized query arguments.
3071    pub fn register_query<F, Fut>(
3072        &mut self,
3073        workflow_type: impl Into<String>,
3074        query_name: impl Into<String>,
3075        handler: F,
3076    ) where
3077        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
3078        Fut: Future<Output = Result<Value>> + Send + 'static,
3079    {
3080        self.queries
3081            .entry(workflow_type.into())
3082            .or_default()
3083            .insert(
3084                query_name.into(),
3085                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
3086            );
3087    }
3088
3089    /// Register a named query against deterministically replayed instance state.
3090    ///
3091    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
3092    /// same state type `S`. The handler receives an immutable, detached state
3093    /// clone, so successful and failed queries cannot affect workflow execution
3094    /// or the state reconstructed by a later query.
3095    pub fn register_replayed_query<S, F, Fut>(
3096        &mut self,
3097        workflow_type: impl Into<String>,
3098        query_name: impl Into<String>,
3099        handler: F,
3100    ) where
3101        S: Clone + Send + Sync + 'static,
3102        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
3103        Fut: Future<Output = Result<Value>> + Send + 'static,
3104    {
3105        let handler = Arc::new(handler);
3106        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
3107            let state = state.downcast::<S>().map_err(|_| {
3108                "registered query state type does not match the replayed workflow state".to_string()
3109            })?;
3110            Ok(Box::pin(handler(ctx, state, args)))
3111        });
3112
3113        self.queries
3114            .entry(workflow_type.into())
3115            .or_default()
3116            .insert(
3117                query_name.into(),
3118                RegisteredQuery::Replayed {
3119                    state_type: TypeId::of::<S>(),
3120                    handler: erased_handler,
3121                },
3122            );
3123    }
3124
3125    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
3126        self.client
3127            .register_worker_with_capabilities(
3128                &self.worker_id,
3129                &self.task_queue,
3130                self.workflows.keys().cloned().collect(),
3131                self.activities.keys().cloned().collect(),
3132                self.max_concurrent_workflow_tasks,
3133                self.max_concurrent_activity_tasks,
3134                (!self.queries.is_empty())
3135                    .then(|| QUERY_TASKS_CAPABILITY.to_string())
3136                    .into_iter()
3137                    .collect(),
3138            )
3139            .await
3140    }
3141
3142    /// Run until shutdown or a terminal worker error occurs.
3143    ///
3144    /// Empty long-poll expirations do not stop the worker. Retryable poll and
3145    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
3146    /// authentication, protocol, and other non-retryable failures are returned.
3147    pub async fn run(&self) -> Result<()> {
3148        self.run_until(std::future::pending::<()>()).await
3149    }
3150
3151    /// Run until `shutdown` resolves or a terminal worker error occurs.
3152    ///
3153    /// This has the same liveness and terminal-error contract as [`Worker::run`].
3154    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
3155    where
3156        F: Future<Output = ()>,
3157    {
3158        let registration = self.register().await?;
3159        let mut heartbeat = tokio::time::interval(Duration::from_secs(
3160            registration
3161                .heartbeat_interval_seconds
3162                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
3163        ));
3164        tokio::pin!(shutdown);
3165        let stop = Arc::new(AtomicBool::new(false));
3166        // Poll responses may already have leased server-side work by the time
3167        // they become ready, so each poller owns its responses through
3168        // completion or failure instead of racing raw polls in this select.
3169        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
3170            let worker = self.clone();
3171            let stop = Arc::clone(&stop);
3172            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
3173        });
3174        let mut activity_poller = (!self.activities.is_empty()).then(|| {
3175            let worker = self.clone();
3176            let stop = Arc::clone(&stop);
3177            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
3178        });
3179        let mut query_poller = (!self.queries.is_empty()).then(|| {
3180            let worker = self.clone();
3181            let stop = Arc::clone(&stop);
3182            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
3183        });
3184
3185        loop {
3186            tokio::select! {
3187                _ = &mut shutdown => {
3188                    stop.store(true, Ordering::SeqCst);
3189                    break;
3190                }
3191                _ = heartbeat.tick() => {
3192                    match self.retry_worker_operation(|| {
3193                        self.client.heartbeat_worker(
3194                            &self.worker_id,
3195                            self.max_concurrent_workflow_tasks,
3196                            self.max_concurrent_activity_tasks,
3197                        )
3198                    }).await
3199                    {
3200                        Ok(acknowledgement) => {
3201                            if let Some(observer) = &self.heartbeat_observer {
3202                                observer(&WorkerHeartbeatObservation {
3203                                    worker_id: self.worker_id.clone(),
3204                                    task_queue: self.task_queue.clone(),
3205                                    acknowledged_at_unix_millis: SystemTime::now()
3206                                        .duration_since(UNIX_EPOCH)
3207                                        .unwrap_or_default()
3208                                        .as_millis()
3209                                        .min(u64::MAX as u128)
3210                                        as u64,
3211                                    acknowledgement,
3212                                });
3213                            }
3214                        }
3215                        Err(error) => {
3216                            stop.store(true, Ordering::SeqCst);
3217                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
3218                            return Err(error);
3219                        }
3220                    }
3221                }
3222                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
3223                    workflow_poller = None;
3224                    let stopped_by_server = stop.load(Ordering::SeqCst);
3225                    stop.store(true, Ordering::SeqCst);
3226                    let poller_result = optional_poller_result("workflow", result);
3227                    let join_result =
3228                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3229                    poller_result?;
3230                    join_result?;
3231                    if stopped_by_server {
3232                        return Ok(());
3233                    }
3234                    return Err(Error::WorkerLoop(
3235                        "workflow poller stopped unexpectedly".to_string(),
3236                    ));
3237                }
3238                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
3239                    activity_poller = None;
3240                    let stopped_by_server = stop.load(Ordering::SeqCst);
3241                    stop.store(true, Ordering::SeqCst);
3242                    let poller_result = optional_poller_result("activity", result);
3243                    let join_result =
3244                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3245                    poller_result?;
3246                    join_result?;
3247                    if stopped_by_server {
3248                        return Ok(());
3249                    }
3250                    return Err(Error::WorkerLoop(
3251                        "activity poller stopped unexpectedly".to_string(),
3252                    ));
3253                }
3254                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
3255                    query_poller = None;
3256                    let stopped_by_server = stop.load(Ordering::SeqCst);
3257                    stop.store(true, Ordering::SeqCst);
3258                    let poller_result = optional_poller_result("query", result);
3259                    let join_result =
3260                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3261                    poller_result?;
3262                    join_result?;
3263                    if stopped_by_server {
3264                        return Ok(());
3265                    }
3266                    return Err(Error::WorkerLoop(
3267                        "query poller stopped unexpectedly".to_string(),
3268                    ));
3269                }
3270            }
3271        }
3272
3273        join_pollers(
3274            workflow_poller.take(),
3275            activity_poller.take(),
3276            query_poller.take(),
3277        )
3278        .await
3279    }
3280
3281    pub async fn run_once(&self) -> Result<usize> {
3282        let mut handled = 0;
3283        match self.poll_workflow_once().await? {
3284            ManagedPollOutcome::Handled => handled += 1,
3285            ManagedPollOutcome::Stop => return Ok(handled),
3286            ManagedPollOutcome::Idle => {}
3287        }
3288        match self.poll_activity_once().await? {
3289            ManagedPollOutcome::Handled => handled += 1,
3290            ManagedPollOutcome::Stop => return Ok(handled),
3291            ManagedPollOutcome::Idle => {}
3292        }
3293        if !self.queries.is_empty() {
3294            match self.poll_query_once().await? {
3295                ManagedPollOutcome::Handled => handled += 1,
3296                ManagedPollOutcome::Stop => return Ok(handled),
3297                ManagedPollOutcome::Idle => {}
3298            }
3299        }
3300        Ok(handled)
3301    }
3302
3303    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
3304        let response = self
3305            .retry_worker_operation(|| {
3306                self.client.poll_workflow_task_response(
3307                    &self.worker_id,
3308                    &self.task_queue,
3309                    self.poll_timeout,
3310                )
3311            })
3312            .await?;
3313        if response.outcome().should_stop() {
3314            return Ok(ManagedPollOutcome::Stop);
3315        }
3316        let Some(task) = response.task else {
3317            return Ok(ManagedPollOutcome::Idle);
3318        };
3319
3320        let task_id = task.task_id.clone();
3321        let attempt = task.workflow_task_attempt;
3322        let lease_owner = task
3323            .lease_owner
3324            .clone()
3325            .unwrap_or_else(|| self.worker_id.clone());
3326
3327        match self.execute_workflow_task(task) {
3328            Ok(commands) if commands.is_empty() => {
3329                // A replay can consume a recorded pending durable command
3330                // without producing a new command. The standalone protocol
3331                // acknowledges that state through the typed waiting outcome;
3332                // an empty completion is rejected by servers that require at
3333                // least one executable command.
3334                self.client
3335                    .fail_workflow_task_with_type(
3336                        &task_id,
3337                        &lease_owner,
3338                        attempt,
3339                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
3340                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
3341                    )
3342                    .await?;
3343            }
3344            Ok(commands) => {
3345                self.client
3346                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
3347                    .await?;
3348            }
3349            Err(error) => {
3350                self.client
3351                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
3352                    .await?;
3353            }
3354        }
3355
3356        Ok(ManagedPollOutcome::Handled)
3357    }
3358
3359    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3360        while !stop.load(Ordering::SeqCst) {
3361            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
3362                stop.store(true, Ordering::SeqCst);
3363                break;
3364            }
3365        }
3366
3367        Ok(())
3368    }
3369
3370    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
3371        let response = self
3372            .retry_worker_operation(|| {
3373                self.client.poll_activity_task_response(
3374                    &self.worker_id,
3375                    &self.task_queue,
3376                    self.poll_timeout,
3377                )
3378            })
3379            .await?;
3380        if response.outcome().should_stop() {
3381            return Ok(ManagedPollOutcome::Stop);
3382        }
3383        let Some(task) = response.task else {
3384            return Ok(ManagedPollOutcome::Idle);
3385        };
3386
3387        let task_id = task.task_id.clone();
3388        let attempt_id = task
3389            .activity_attempt_id
3390            .clone()
3391            .or(task.attempt_id.clone())
3392            .unwrap_or_default();
3393        let lease_owner = task
3394            .lease_owner
3395            .clone()
3396            .unwrap_or_else(|| self.worker_id.clone());
3397        let codec = task.payload_codec.clone();
3398        let result = self.execute_activity_task(task).await;
3399        match result {
3400            Ok(value) => {
3401                let completion = self
3402                    .client
3403                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
3404                    .await;
3405                if let Err(error) = completion {
3406                    if !activity_task_rejection_is_final(&error) {
3407                        return Err(error);
3408                    }
3409                }
3410            }
3411            Err(error) => {
3412                let failure = self
3413                    .client
3414                    .fail_activity_task(
3415                        &task_id,
3416                        &attempt_id,
3417                        &lease_owner,
3418                        error.to_string(),
3419                        false,
3420                    )
3421                    .await;
3422                if let Err(error) = failure {
3423                    if !activity_task_rejection_is_final(&error) {
3424                        return Err(error);
3425                    }
3426                }
3427            }
3428        }
3429
3430        Ok(ManagedPollOutcome::Handled)
3431    }
3432
3433    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3434        while !stop.load(Ordering::SeqCst) {
3435            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
3436                stop.store(true, Ordering::SeqCst);
3437                break;
3438            }
3439        }
3440
3441        Ok(())
3442    }
3443
3444    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
3445        let response = self
3446            .retry_worker_operation(|| {
3447                self.client.poll_query_task_response(
3448                    &self.worker_id,
3449                    &self.task_queue,
3450                    self.poll_timeout,
3451                )
3452            })
3453            .await?;
3454        if response.outcome().should_stop() {
3455            return Ok(ManagedPollOutcome::Stop);
3456        }
3457        let Some(task) = response.task else {
3458            return Ok(ManagedPollOutcome::Idle);
3459        };
3460
3461        let query_task_id = task.query_task_id.clone();
3462        let attempt = task.query_task_attempt;
3463        let lease_owner = task
3464            .lease_owner
3465            .clone()
3466            .unwrap_or_else(|| self.worker_id.clone());
3467        let codec = task.payload_codec.clone();
3468
3469        match self.execute_query_task(task).await {
3470            Ok(value) => {
3471                let result_envelope = match encode_value_envelope(&value, &codec) {
3472                    Ok(result_envelope) => result_envelope,
3473                    Err(error) => {
3474                        let failure = self
3475                            .client
3476                            .fail_query_task(
3477                                &query_task_id,
3478                                &lease_owner,
3479                                attempt,
3480                                error.to_string(),
3481                                "query_result_encode_failed",
3482                                "QueryResultEncodeFailed",
3483                            )
3484                            .await;
3485                        if let Err(error) = failure {
3486                            if !query_task_rejection_is_final(&error) {
3487                                return Err(error);
3488                            }
3489                        }
3490                        return Ok(ManagedPollOutcome::Handled);
3491                    }
3492                };
3493
3494                if let Err(error) = self
3495                    .client
3496                    .complete_query_task_with_envelope(
3497                        &query_task_id,
3498                        &lease_owner,
3499                        attempt,
3500                        value,
3501                        result_envelope,
3502                    )
3503                    .await
3504                {
3505                    if !query_task_rejection_is_final(&error) {
3506                        return Err(error);
3507                    }
3508                }
3509            }
3510            Err(failure) => {
3511                let result = self
3512                    .client
3513                    .fail_query_task(
3514                        &query_task_id,
3515                        &lease_owner,
3516                        attempt,
3517                        failure.message,
3518                        failure.reason,
3519                        failure.failure_type,
3520                    )
3521                    .await;
3522                if let Err(error) = result {
3523                    if !query_task_rejection_is_final(&error) {
3524                        return Err(error);
3525                    }
3526                }
3527            }
3528        }
3529
3530        Ok(ManagedPollOutcome::Handled)
3531    }
3532
3533    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3534        while !stop.load(Ordering::SeqCst) {
3535            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
3536                stop.store(true, Ordering::SeqCst);
3537                break;
3538            }
3539        }
3540
3541        Ok(())
3542    }
3543
3544    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
3545    where
3546        F: FnMut() -> Fut,
3547        Fut: Future<Output = Result<T>>,
3548    {
3549        let mut retries = 0;
3550
3551        loop {
3552            match operation().await {
3553                Err(error)
3554                    if worker_operation_is_retryable(&error)
3555                        && retries < self.retry_policy.max_retries =>
3556                {
3557                    retries += 1;
3558                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
3559                }
3560                result => return result,
3561            }
3562        }
3563    }
3564
3565    async fn execute_query_task(
3566        &self,
3567        mut task: QueryTask,
3568    ) -> std::result::Result<Value, QueryTaskExecutionFailure> {
3569        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
3570            return Err(QueryTaskExecutionFailure::new(
3571                "query_payload_decode_failed",
3572                format!(
3573                    "cannot decode query payload with unsupported codec {:?}",
3574                    task.payload_codec
3575                ),
3576                "QueryPayloadDecodeFailed",
3577            ));
3578        }
3579
3580        if !self.workflows.contains_key(&task.workflow_type) {
3581            return Err(QueryTaskExecutionFailure::new(
3582                "query_workflow_type_not_registered",
3583                format!("no workflow registered for type {:?}", task.workflow_type),
3584                "WorkflowTypeNotRegistered",
3585            ));
3586        }
3587
3588        let Some(handlers) = self.queries.get(&task.workflow_type) else {
3589            return Err(QueryTaskExecutionFailure::new(
3590                "query_handler_unavailable",
3591                format!(
3592                    "query handlers are unavailable for workflow type {:?}",
3593                    task.workflow_type
3594                ),
3595                "QueryHandlerUnavailable",
3596            ));
3597        };
3598        let Some(query) = handlers.get(&task.query_name) else {
3599            return Err(QueryTaskExecutionFailure::new(
3600                "rejected_unknown_query",
3601                format!("unknown query {:?}", task.query_name),
3602                "QueryFailed",
3603            ));
3604        };
3605
3606        let args = decode_task_arguments(task.query_arguments.as_ref(), &task.payload_codec)
3607            .map_err(|error| {
3608                QueryTaskExecutionFailure::new(
3609                    "query_payload_decode_failed",
3610                    format!("cannot decode query arguments: {error}"),
3611                    "QueryPayloadDecodeFailed",
3612                )
3613            })?;
3614        let workflow_input =
3615            decode_task_arguments(task.workflow_arguments.as_ref(), &task.payload_codec).map_err(
3616                |error| {
3617                    QueryTaskExecutionFailure::new(
3618                        "query_workflow_state_unavailable",
3619                        format!("cannot decode workflow start input: {error}"),
3620                        "QueryWorkflowStateUnavailable",
3621                    )
3622                },
3623            )?;
3624        hydrate_query_history_from_export(&mut task).map_err(|error| {
3625            QueryTaskExecutionFailure::new(
3626                "query_workflow_state_unavailable",
3627                format!("cannot restore query history snapshot: {error}"),
3628                "QueryWorkflowStateUnavailable",
3629            )
3630        })?;
3631        enrich_query_history_from_export(&mut task).map_err(|error| {
3632            QueryTaskExecutionFailure::new(
3633                "query_workflow_state_unavailable",
3634                format!("cannot restore compact query history payloads: {error}"),
3635                "QueryWorkflowStateUnavailable",
3636            )
3637        })?;
3638        let signal_events = query_signal_events(&task).map_err(|error| {
3639            QueryTaskExecutionFailure::new(
3640                "query_workflow_state_unavailable",
3641                format!("cannot decode committed workflow signals: {error}"),
3642                "QueryWorkflowStateUnavailable",
3643            )
3644        })?;
3645        let history_events = Arc::new(std::mem::take(&mut task.history_events));
3646        let context = QueryContext {
3647            workflow_id: task.workflow_id,
3648            run_id: task.run_id,
3649            workflow_type: task.workflow_type.clone(),
3650            run_status: task.run_status,
3651            workflow_input,
3652            history_events: Arc::clone(&history_events),
3653            signal_events: Arc::new(signal_events),
3654        };
3655
3656        let future = match query {
3657            RegisteredQuery::Snapshot(handler) => handler(context, args),
3658            RegisteredQuery::Replayed {
3659                state_type,
3660                handler,
3661            } => {
3662                let workflow = self
3663                    .workflows
3664                    .get(&task.workflow_type)
3665                    .expect("workflow registration was checked above");
3666                if workflow.state_type != Some(*state_type) {
3667                    return Err(QueryTaskExecutionFailure::new(
3668                        "query_workflow_state_unavailable",
3669                        "replayed query state type does not match its workflow registration",
3670                        "QueryWorkflowStateUnavailable",
3671                    ));
3672                }
3673                let replay = workflow.replay.as_ref().ok_or_else(|| {
3674                    QueryTaskExecutionFailure::new(
3675                        "query_workflow_state_unavailable",
3676                        format!(
3677                            "workflow type {:?} is not registered for instance-state replay",
3678                            task.workflow_type
3679                        ),
3680                        "QueryWorkflowStateUnavailable",
3681                    )
3682                })?;
3683                let workflow_state = Arc::new(Mutex::new(
3684                    WorkflowState::new_with_identity(
3685                        history_events.as_ref().clone(),
3686                        context.workflow_id.clone(),
3687                        context.run_id.clone(),
3688                        self.task_queue.clone(),
3689                        task.payload_codec,
3690                        None,
3691                    )
3692                    .map_err(|error| {
3693                        QueryTaskExecutionFailure::new(
3694                            "query_workflow_state_unavailable",
3695                            format!("workflow replay failed before query: {error}"),
3696                            "QueryWorkflowStateUnavailable",
3697                        )
3698                    })?,
3699                ));
3700                let workflow_context = WorkflowContext {
3701                    state: workflow_state,
3702                };
3703                let mut invocation =
3704                    replay(workflow_context.clone(), context.workflow_input.clone());
3705                let mut cx = TaskContext::from_waker(noop_waker_ref());
3706                match invocation.future.as_mut().poll(&mut cx) {
3707                    Poll::Ready(Ok(_)) => {
3708                        workflow_context
3709                            .ensure_history_consumed()
3710                            .map_err(|error| {
3711                                QueryTaskExecutionFailure::new(
3712                                    "query_workflow_state_unavailable",
3713                                    format!("workflow replay failed before query: {error}"),
3714                                    "QueryWorkflowStateUnavailable",
3715                                )
3716                            })?;
3717                    }
3718                    Poll::Ready(Err(error)) => {
3719                        return Err(QueryTaskExecutionFailure::new(
3720                            "query_workflow_state_unavailable",
3721                            format!("workflow replay failed before query: {error}"),
3722                            "QueryWorkflowStateUnavailable",
3723                        ));
3724                    }
3725                    Poll::Pending => {
3726                        let commands = workflow_context.take_commands().map_err(|error| {
3727                            QueryTaskExecutionFailure::new(
3728                                "query_workflow_state_unavailable",
3729                                format!("workflow replay failed before query: {error}"),
3730                                "QueryWorkflowStateUnavailable",
3731                            )
3732                        })?;
3733                        if commands.is_empty()
3734                            && !workflow_context
3735                                .matched_recorded_pending()
3736                                .map_err(|error| {
3737                                    QueryTaskExecutionFailure::new(
3738                                        "query_workflow_state_unavailable",
3739                                        format!("workflow replay failed before query: {error}"),
3740                                        "QueryWorkflowStateUnavailable",
3741                                    )
3742                                })?
3743                        {
3744                            return Err(QueryTaskExecutionFailure::new(
3745                                "query_workflow_state_unavailable",
3746                                "workflow replay yielded without a durable command",
3747                                "QueryWorkflowStateUnavailable",
3748                            ));
3749                        }
3750                    }
3751                }
3752                let state = (invocation.snapshot)().map_err(|error| {
3753                    QueryTaskExecutionFailure::new(
3754                        "query_workflow_state_unavailable",
3755                        format!("cannot snapshot replayed workflow state: {error}"),
3756                        "QueryWorkflowStateUnavailable",
3757                    )
3758                })?;
3759                handler(context, state, args).map_err(|message| {
3760                    QueryTaskExecutionFailure::new(
3761                        "query_workflow_state_unavailable",
3762                        message,
3763                        "QueryWorkflowStateUnavailable",
3764                    )
3765                })?
3766            }
3767        };
3768
3769        future.await.map_err(|error| {
3770            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
3771        })
3772    }
3773
3774    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
3775        let workflow = self
3776            .workflows
3777            .get(&task.workflow_type)
3778            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
3779        let input = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3780        let resume_signal = decode_resume_signal(&task)?;
3781        let state = Arc::new(Mutex::new(WorkflowState::new_with_identity(
3782            task.history_events,
3783            task.workflow_id,
3784            task.run_id,
3785            self.task_queue.clone(),
3786            task.payload_codec.clone(),
3787            resume_signal,
3788        )?));
3789        let ctx = WorkflowContext { state };
3790        let mut future = (workflow.execute)(ctx.clone(), input);
3791        let mut cx = TaskContext::from_waker(noop_waker_ref());
3792
3793        match future.as_mut().poll(&mut cx) {
3794            Poll::Ready(Ok(result)) => {
3795                ctx.ensure_history_consumed()?;
3796                let result = encode_value_envelope(&result, &task.payload_codec)?;
3797                Ok(vec![json!({
3798                    "type": "complete_workflow",
3799                    "result": result
3800                })])
3801            }
3802            Poll::Ready(Err(error @ Error::ChildWorkflowFailed(_))) => {
3803                Ok(vec![workflow_failure_command(&error)])
3804            }
3805            Poll::Ready(Err(error)) => Err(error),
3806            Poll::Pending => {
3807                let commands = ctx.take_commands()?;
3808                if commands.is_empty() && !ctx.matched_recorded_pending()? {
3809                    Err(Error::WorkflowYieldedWithoutCommand)
3810                } else {
3811                    Ok(commands)
3812                }
3813            }
3814        }
3815    }
3816
3817    async fn execute_activity_task(&self, task: ActivityTask) -> Result<Value> {
3818        let handler = self
3819            .activities
3820            .get(&task.activity_type)
3821            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
3822        let args = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3823        let attempt_id = task
3824            .activity_attempt_id
3825            .clone()
3826            .or(task.attempt_id.clone())
3827            .unwrap_or_default();
3828        let lease_owner = task
3829            .lease_owner
3830            .clone()
3831            .unwrap_or_else(|| self.worker_id.clone());
3832        let ctx = ActivityContext {
3833            client: self.client.clone(),
3834            task_id: task.task_id,
3835            activity_attempt_id: attempt_id,
3836            lease_owner,
3837            activity_type: task.activity_type,
3838            attempt_number: task.attempt_number,
3839            task_queue: self.task_queue.clone(),
3840            worker_id: self.worker_id.clone(),
3841        };
3842
3843        handler(ctx, args).await
3844    }
3845}
3846
3847fn poller_result(
3848    kind: &str,
3849    result: std::result::Result<Result<()>, tokio::task::JoinError>,
3850) -> Result<()> {
3851    match result {
3852        Ok(result) => result,
3853        Err(error) => Err(Error::WorkerLoop(format!(
3854            "{kind} poller join error: {error}"
3855        ))),
3856    }
3857}
3858
3859fn optional_poller_result(
3860    kind: &str,
3861    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
3862) -> Result<()> {
3863    match result {
3864        Some(result) => poller_result(kind, result),
3865        None => Ok(()),
3866    }
3867}
3868
3869async fn join_pollers(
3870    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3871    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3872    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3873) -> Result<()> {
3874    let mut first_error = None;
3875
3876    if let Some(handle) = workflow_poller {
3877        if let Err(error) = poller_result("workflow", handle.await) {
3878            first_error.get_or_insert(error);
3879        }
3880    }
3881
3882    if let Some(handle) = activity_poller {
3883        if let Err(error) = poller_result("activity", handle.await) {
3884            first_error.get_or_insert(error);
3885        }
3886    }
3887
3888    if let Some(handle) = query_poller {
3889        if let Err(error) = poller_result("query", handle.await) {
3890            first_error.get_or_insert(error);
3891        }
3892    }
3893
3894    if let Some(error) = first_error {
3895        Err(error)
3896    } else {
3897        Ok(())
3898    }
3899}
3900
3901fn default_worker_id() -> String {
3902    let millis = SystemTime::now()
3903        .duration_since(UNIX_EPOCH)
3904        .unwrap_or_default()
3905        .as_millis();
3906    format!("rust-worker-{}-{millis}", std::process::id())
3907}
3908
3909fn unique_request_id(prefix: &str) -> String {
3910    let nanos = SystemTime::now()
3911        .duration_since(UNIX_EPOCH)
3912        .unwrap_or_default()
3913        .as_nanos();
3914    format!("{prefix}-{}-{nanos}", std::process::id())
3915}
3916
3917#[derive(Debug)]
3918struct QueryTaskExecutionFailure {
3919    reason: String,
3920    message: String,
3921    failure_type: String,
3922}
3923
3924impl QueryTaskExecutionFailure {
3925    fn new(
3926        reason: impl Into<String>,
3927        message: impl Into<String>,
3928        failure_type: impl Into<String>,
3929    ) -> Self {
3930        Self {
3931            reason: reason.into(),
3932            message: message.into(),
3933            failure_type: failure_type.into(),
3934        }
3935    }
3936}
3937
3938/// Typed local state owned by one deterministic workflow invocation.
3939///
3940/// Use [`WorkflowInstance::update`] for the same state transitions during
3941/// ordinary execution and replay. A replayed query receives a detached
3942/// immutable `Arc<S>` rather than this mutation-capable handle.
3943#[derive(Clone, Debug)]
3944pub struct WorkflowInstance<S> {
3945    state: Arc<Mutex<S>>,
3946}
3947
3948impl<S> WorkflowInstance<S> {
3949    fn new(state: S) -> Self {
3950        Self {
3951            state: Arc::new(Mutex::new(state)),
3952        }
3953    }
3954
3955    /// Read the current workflow-instance state without changing it.
3956    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
3957        let state = self
3958            .state
3959            .lock()
3960            .map_err(|_| Error::WorkflowStatePoisoned)?;
3961        Ok(reader(&state))
3962    }
3963
3964    /// Apply one deterministic workflow-instance state transition.
3965    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
3966        let mut state = self
3967            .state
3968            .lock()
3969            .map_err(|_| Error::WorkflowStatePoisoned)?;
3970        Ok(transition(&mut state))
3971    }
3972}
3973
3974impl<S: Clone> WorkflowInstance<S> {
3975    fn snapshot(&self) -> Result<S> {
3976        self.read(Clone::clone)
3977    }
3978}
3979
3980#[derive(Clone, Debug)]
3981pub struct WorkflowContext {
3982    state: Arc<Mutex<WorkflowState>>,
3983}
3984
3985impl WorkflowContext {
3986    /// Identity of the parent workflow currently being replayed.
3987    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
3988        let state = self
3989            .state
3990            .lock()
3991            .map_err(|_| Error::WorkflowStatePoisoned)?;
3992        Ok(WorkflowIdentity {
3993            workflow_id: state.workflow_id.clone(),
3994            run_id: state.run_id.clone(),
3995        })
3996    }
3997
3998    pub fn activity<T: Serialize>(
3999        &self,
4000        activity_type: impl Into<String>,
4001        args: T,
4002    ) -> ActivityCall {
4003        self.activity_with_options(activity_type, ActivityOptions::new(), args)
4004    }
4005
4006    pub fn activity_on_queue<T, Q>(
4007        &self,
4008        activity_type: impl Into<String>,
4009        task_queue: Option<Q>,
4010        args: T,
4011    ) -> ActivityCall
4012    where
4013        T: Serialize,
4014        Q: Into<String>,
4015    {
4016        let mut options = ActivityOptions::new();
4017        options.task_queue = task_queue.map(Into::into);
4018        self.activity_with_options(activity_type, options, args)
4019    }
4020
4021    /// Schedule one durable activity with retry, routing, and timeout options.
4022    ///
4023    /// Options are validated before the command is emitted. Once the command is
4024    /// recorded, replay consumes the same activity lifecycle at this command
4025    /// position and never emits a duplicate schedule.
4026    ///
4027    /// ```no_run
4028    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
4029    /// # use std::time::Duration;
4030    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
4031    /// let result = ctx
4032    ///     .activity_with_options(
4033    ///         "charge-card",
4034    ///         ActivityOptions::new()
4035    ///             .task_queue("payments")
4036    ///             .retry_policy(
4037    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
4038    ///                     Duration::from_secs(1),
4039    ///                     2,
4040    ///                     Some(Duration::from_secs(30)),
4041    ///                 ),
4042    ///             )
4043    ///             .start_to_close_timeout(Duration::from_secs(60))
4044    ///             .schedule_to_close_timeout(Duration::from_secs(180))
4045    ///             .heartbeat_timeout(Duration::from_secs(15)),
4046    ///         json!([{"order_id": "order-42"}]),
4047    ///     )
4048    ///     .await;
4049    /// match result {
4050    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
4051    ///         "reason": failure.reason,
4052    ///         "timeout_kind": failure.timeout_kind,
4053    ///     })),
4054    ///     other => other,
4055    /// }
4056    /// # }
4057    /// ```
4058    pub fn activity_with_options<T: Serialize>(
4059        &self,
4060        activity_type: impl Into<String>,
4061        options: ActivityOptions,
4062        args: T,
4063    ) -> ActivityCall {
4064        ActivityCall {
4065            ctx: self.clone(),
4066            activity_type: activity_type.into(),
4067            options,
4068            args: Some(serde_json::to_value(args).map_err(Error::from)),
4069            scheduled: false,
4070        }
4071    }
4072
4073    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
4074        SignalCall {
4075            ctx: self.clone(),
4076            signal_name: signal_name.into(),
4077            opened_wait: false,
4078            matched_pending: false,
4079        }
4080    }
4081
4082    /// Wait for server-backed durable time without blocking the worker executor.
4083    ///
4084    /// Polling this future emits one `start_timer` command and yields. The
4085    /// server records the deadline, so neither worker nor server restarts reset
4086    /// the wait. Replay resolves the future only from a `TimerScheduled` and
4087    /// `TimerFired` pair at the same position in the shared durable-command
4088    /// stream, with matching sequence, timer identity, and delay. Sub-second
4089    /// durations round up because protocol deadlines use whole seconds.
4090    ///
4091    /// ```no_run
4092    /// # use durable_workflow::{json, Client, Worker};
4093    /// # use std::time::Duration;
4094    /// # fn configure(client: Client) {
4095    /// let mut worker = Worker::new(client, "rust-workers");
4096    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
4097    ///     ctx.sleep(Duration::from_secs(5)).await?;
4098    ///     Ok(json!({"status": "timer fired"}))
4099    /// });
4100    /// # }
4101    /// ```
4102    pub fn sleep(&self, duration: Duration) -> TimerCall {
4103        let delay_seconds = duration
4104            .as_secs()
4105            .checked_add(u64::from(duration.subsec_nanos() > 0));
4106        TimerCall {
4107            ctx: self.clone(),
4108            delay_seconds,
4109            scheduled: false,
4110            matched_pending: false,
4111        }
4112    }
4113
4114    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
4115    pub fn start_timer(&self, duration: Duration) -> TimerCall {
4116        self.sleep(duration)
4117    }
4118
4119    /// Start a named durable child on an explicit queue and await its result.
4120    ///
4121    /// The command is recorded in the parent's sequence-ordered durable command
4122    /// stream. Replay keeps a scheduled child pending without emitting another
4123    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
4124    /// values preserve the history payload codec and include both sides of the
4125    /// durable relationship; failures are returned as
4126    /// [`Error::ChildWorkflowFailed`].
4127    ///
4128    /// ```no_run
4129    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
4130    /// # fn configure(client: Client) {
4131    /// let mut worker = Worker::new(client, "parent-workers");
4132    /// worker.register_workflow("order-parent", |ctx, _input| async move {
4133    ///     let child = ctx
4134    ///         .start_child_workflow(
4135    ///             "fulfil-order",
4136    ///             ChildWorkflowOptions::new("fulfilment-workers")
4137    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
4138    ///             json!([{"order_id": "order-42"}]),
4139    ///         )
4140    ///         .await?;
4141    ///     Ok(child.result)
4142    /// });
4143    /// # }
4144    /// ```
4145    pub fn start_child_workflow<T: Serialize>(
4146        &self,
4147        workflow_type: impl Into<String>,
4148        options: ChildWorkflowOptions,
4149        args: T,
4150    ) -> ChildWorkflowCall {
4151        ChildWorkflowCall {
4152            ctx: self.clone(),
4153            workflow_type: workflow_type.into(),
4154            options,
4155            args: Some(serde_json::to_value(args).map_err(Error::from)),
4156            scheduled: false,
4157            matched_pending: false,
4158        }
4159    }
4160
4161    fn take_commands(&self) -> Result<Vec<Value>> {
4162        let mut state = self
4163            .state
4164            .lock()
4165            .map_err(|_| Error::WorkflowStatePoisoned)?;
4166        Ok(std::mem::take(&mut state.commands))
4167    }
4168
4169    fn matched_recorded_pending(&self) -> Result<bool> {
4170        let state = self
4171            .state
4172            .lock()
4173            .map_err(|_| Error::WorkflowStatePoisoned)?;
4174        Ok(state.matched_recorded_pending)
4175    }
4176
4177    fn ensure_history_consumed(&self) -> Result<()> {
4178        let state = self
4179            .state
4180            .lock()
4181            .map_err(|_| Error::WorkflowStatePoisoned)?;
4182        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
4183            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
4184                "recorded_commands_unconsumed",
4185                Some(command.sequence()),
4186                Some(command.shape().to_string()),
4187                Some("workflow completion".to_string()),
4188                "workflow completed before consuming all recorded durable commands",
4189            )));
4190        }
4191        Ok(())
4192    }
4193}
4194
4195#[derive(Debug)]
4196struct WorkflowState {
4197    history: Vec<HistoryEvent>,
4198    workflow_id: Option<String>,
4199    run_id: Option<String>,
4200    task_queue: String,
4201    payload_codec: String,
4202    resume_signal: Option<ResumeSignal>,
4203    recorded_commands: Vec<RecordedCommand>,
4204    command_cursor: usize,
4205    matched_recorded_pending: bool,
4206    signal_cursors: HashMap<String, usize>,
4207    commands: Vec<Value>,
4208}
4209
4210impl WorkflowState {
4211    #[cfg(test)]
4212    fn new(
4213        history: Vec<HistoryEvent>,
4214        task_queue: String,
4215        payload_codec: String,
4216        resume_signal: Option<ResumeSignal>,
4217    ) -> Result<Self> {
4218        Self::new_with_identity(
4219            history,
4220            None,
4221            None,
4222            task_queue,
4223            payload_codec,
4224            resume_signal,
4225        )
4226    }
4227
4228    fn new_with_identity(
4229        history: Vec<HistoryEvent>,
4230        workflow_id: Option<String>,
4231        run_id: Option<String>,
4232        task_queue: String,
4233        payload_codec: String,
4234        resume_signal: Option<ResumeSignal>,
4235    ) -> Result<Self> {
4236        let recorded_commands = recorded_commands(
4237            &history,
4238            &payload_codec,
4239            WorkflowIdentity {
4240                workflow_id: workflow_id.clone(),
4241                run_id: run_id.clone(),
4242            },
4243        )?;
4244        Ok(Self {
4245            history,
4246            workflow_id,
4247            run_id,
4248            task_queue,
4249            payload_codec,
4250            resume_signal,
4251            recorded_commands,
4252            command_cursor: 0,
4253            matched_recorded_pending: false,
4254            signal_cursors: HashMap::new(),
4255            commands: Vec::new(),
4256        })
4257    }
4258}
4259
4260#[derive(Clone, Debug)]
4261enum RecordedCommand {
4262    Activity {
4263        sequence: u64,
4264        activity_type: Option<String>,
4265        options: Option<RecordedActivityOptions>,
4266        outcome: Option<ActivityOutcome>,
4267    },
4268    Timer {
4269        sequence: u64,
4270        delay_seconds: u64,
4271        fired: bool,
4272    },
4273    ChildWorkflow {
4274        sequence: u64,
4275        workflow_type: Option<String>,
4276        outcome: Option<ChildWorkflowOutcome>,
4277    },
4278    SignalWait {
4279        sequence: u64,
4280        signal_name: Option<String>,
4281    },
4282}
4283
4284#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4285struct RecordedActivityOptions {
4286    task_queue: RecordedSnapshotValue<Option<String>>,
4287    execution_mode: RecordedSnapshotValue<Option<String>>,
4288    retry_policy: ActivityRetrySnapshot,
4289}
4290
4291#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4292enum RecordedSnapshotValue<T> {
4293    /// Older history did not persist this field, so it cannot constrain replay.
4294    Unknown,
4295    Known(T),
4296}
4297
4298impl<T: PartialEq> RecordedSnapshotValue<T> {
4299    fn matches_current(&self, current: &Self) -> bool {
4300        match self {
4301            Self::Unknown => true,
4302            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
4303        }
4304    }
4305}
4306
4307#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4308struct ActivityRetrySnapshot {
4309    snapshot_version: RecordedSnapshotValue<Option<u64>>,
4310    max_attempts: RecordedSnapshotValue<Option<u64>>,
4311    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
4312    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4313    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
4314    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4315    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
4316    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
4317}
4318
4319impl ActivityRetrySnapshot {
4320    fn matches_current(&self, current: &Self) -> bool {
4321        self.snapshot_version
4322            .matches_current(&current.snapshot_version)
4323            && self.max_attempts.matches_current(&current.max_attempts)
4324            && self
4325                .backoff_seconds
4326                .matches_current(&current.backoff_seconds)
4327            && self
4328                .start_to_close_timeout
4329                .matches_current(&current.start_to_close_timeout)
4330            && self
4331                .schedule_to_start_timeout
4332                .matches_current(&current.schedule_to_start_timeout)
4333            && self
4334                .schedule_to_close_timeout
4335                .matches_current(&current.schedule_to_close_timeout)
4336            && self
4337                .heartbeat_timeout
4338                .matches_current(&current.heartbeat_timeout)
4339            && self
4340                .non_retryable_error_types
4341                .matches_current(&current.non_retryable_error_types)
4342    }
4343}
4344
4345fn recorded_optional_u64(
4346    object: Option<&serde_json::Map<String, Value>>,
4347    field: &str,
4348) -> RecordedSnapshotValue<Option<u64>> {
4349    match object.and_then(|object| object.get(field)) {
4350        None => RecordedSnapshotValue::Unknown,
4351        Some(Value::Null) => RecordedSnapshotValue::Known(None),
4352        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
4353    }
4354}
4355
4356fn recorded_optional_string(
4357    object: &serde_json::Map<String, Value>,
4358    field: &str,
4359) -> RecordedSnapshotValue<Option<String>> {
4360    match object.get(field) {
4361        None => RecordedSnapshotValue::Unknown,
4362        Some(Value::Null) => RecordedSnapshotValue::Known(None),
4363        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
4364    }
4365}
4366
4367fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
4368    let policy = policy.and_then(Value::as_object);
4369    let backoff_seconds = policy
4370        .and_then(|policy| policy.get("backoff_seconds"))
4371        .and_then(Value::as_array)
4372        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4373        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
4374    let mut non_retryable_error_types = Vec::new();
4375    for error_type in policy
4376        .and_then(|policy| policy.get("non_retryable_error_types"))
4377        .and_then(Value::as_array)
4378        .into_iter()
4379        .flatten()
4380        .filter_map(Value::as_str)
4381        .map(str::trim)
4382        .filter(|error_type| !error_type.is_empty())
4383    {
4384        if !non_retryable_error_types
4385            .iter()
4386            .any(|recorded| recorded == error_type)
4387        {
4388            non_retryable_error_types.push(error_type.to_string());
4389        }
4390    }
4391
4392    ActivityRetrySnapshot {
4393        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
4394        max_attempts: recorded_optional_u64(policy, "max_attempts"),
4395        backoff_seconds,
4396        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
4397        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
4398        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
4399        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
4400        non_retryable_error_types: if policy
4401            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
4402        {
4403            RecordedSnapshotValue::Known(non_retryable_error_types)
4404        } else {
4405            RecordedSnapshotValue::Unknown
4406        },
4407    }
4408}
4409
4410fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
4411    let policy = options.retry_policy.as_ref();
4412    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
4413        Some(Value::Null) => None,
4414        Some(value) => value_as_u64(value),
4415        None => Some(1),
4416    };
4417    let backoff_seconds = policy
4418        .and_then(|policy| policy.get("backoff_seconds"))
4419        .and_then(Value::as_array)
4420        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4421        .unwrap_or_default();
4422    let non_retryable_error_types = policy
4423        .and_then(|policy| policy.get("non_retryable_error_types"))
4424        .and_then(Value::as_array)
4425        .into_iter()
4426        .flatten()
4427        .filter_map(Value::as_str)
4428        .map(str::to_string)
4429        .collect();
4430
4431    ActivityRetrySnapshot {
4432        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
4433        max_attempts: RecordedSnapshotValue::Known(max_attempts),
4434        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
4435        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
4436        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
4437        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
4438        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
4439        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
4440    }
4441}
4442
4443fn activity_options_description(options: &RecordedActivityOptions) -> String {
4444    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
4445}
4446
4447impl RecordedCommand {
4448    fn sequence(&self) -> u64 {
4449        match self {
4450            Self::Activity { sequence, .. }
4451            | Self::Timer { sequence, .. }
4452            | Self::ChildWorkflow { sequence, .. }
4453            | Self::SignalWait { sequence, .. } => *sequence,
4454        }
4455    }
4456
4457    fn shape(&self) -> &'static str {
4458        match self {
4459            Self::Activity { .. } => "activity",
4460            Self::Timer { .. } => "timer",
4461            Self::ChildWorkflow { .. } => "child workflow",
4462            Self::SignalWait { .. } => "signal wait",
4463        }
4464    }
4465}
4466
4467#[derive(Clone, Debug)]
4468struct ResumeSignal {
4469    signal_id: Option<String>,
4470    signal_name: String,
4471    arguments: Vec<Value>,
4472}
4473
4474pub struct ActivityCall {
4475    ctx: WorkflowContext,
4476    activity_type: String,
4477    options: ActivityOptions,
4478    args: Option<Result<Value>>,
4479    scheduled: bool,
4480}
4481
4482impl Future for ActivityCall {
4483    type Output = Result<Value>;
4484
4485    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4486        let ctx = self.ctx.clone();
4487        let mut state = match ctx.state.lock() {
4488            Ok(state) => state,
4489            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4490        };
4491
4492        if self.scheduled {
4493            return Poll::Pending;
4494        }
4495
4496        let options = match self.options.validate() {
4497            Ok(options) => options,
4498            Err(error) => {
4499                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
4500            }
4501        };
4502        let task_queue = options
4503            .task_queue
4504            .clone()
4505            .unwrap_or_else(|| state.task_queue.clone());
4506        let current_recorded_options = RecordedActivityOptions {
4507            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
4508            // Rust schedules ordinary durable activities. The server records a
4509            // non-null mode only for a specialized execution primitive.
4510            execution_mode: RecordedSnapshotValue::Known(None),
4511            retry_policy: current_activity_retry_snapshot(&options),
4512        };
4513
4514        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4515            let sequence = recorded.sequence();
4516            match recorded {
4517                RecordedCommand::Activity {
4518                    activity_type,
4519                    options: recorded_options,
4520                    outcome,
4521                    ..
4522                } => {
4523                    if let Some(recorded_type) = activity_type {
4524                        if recorded_type != self.activity_type {
4525                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4526                                ReplayFailure::new(
4527                                    "recorded_command_detail_mismatch",
4528                                    Some(sequence),
4529                                    Some(format!("activity:{recorded_type}")),
4530                                    Some(format!("activity:{}", self.activity_type)),
4531                                    "recorded activity type differs from the current workflow command",
4532                                ),
4533                            )));
4534                        }
4535                    }
4536                    if let Some(recorded_options) = recorded_options {
4537                        if !recorded_options
4538                            .task_queue
4539                            .matches_current(&current_recorded_options.task_queue)
4540                        {
4541                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4542                                ReplayFailure::new(
4543                                    "activity_task_queue_mismatch",
4544                                    Some(sequence),
4545                                    Some(activity_options_description(&recorded_options)),
4546                                    Some(activity_options_description(&current_recorded_options)),
4547                                    "recorded activity task queue differs from the current workflow command",
4548                                ),
4549                            )));
4550                        }
4551                        if !recorded_options
4552                            .execution_mode
4553                            .matches_current(&current_recorded_options.execution_mode)
4554                        {
4555                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4556                                ReplayFailure::new(
4557                                    "activity_execution_mode_mismatch",
4558                                    Some(sequence),
4559                                    Some(activity_options_description(&recorded_options)),
4560                                    Some(activity_options_description(&current_recorded_options)),
4561                                    "recorded activity execution mode differs from the current workflow command",
4562                                ),
4563                            )));
4564                        }
4565                        if !recorded_options
4566                            .retry_policy
4567                            .matches_current(&current_recorded_options.retry_policy)
4568                        {
4569                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4570                                ReplayFailure::new(
4571                                    "activity_retry_policy_mismatch",
4572                                    Some(sequence),
4573                                    Some(activity_options_description(&recorded_options)),
4574                                    Some(activity_options_description(&current_recorded_options)),
4575                                    "recorded activity retry policy differs from the current workflow command",
4576                                ),
4577                            )));
4578                        }
4579                    }
4580                    state.command_cursor += 1;
4581                    if let Some(outcome) = outcome {
4582                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
4583                    }
4584                    state.matched_recorded_pending = true;
4585                    self.scheduled = true;
4586                    return Poll::Pending;
4587                }
4588                other => {
4589                    return Poll::Ready(Err(command_mismatch(
4590                        &other,
4591                        format!("activity:{}", self.activity_type),
4592                    )));
4593                }
4594            }
4595        }
4596
4597        if !self.scheduled {
4598            let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4599                Ok(args) => args,
4600                Err(error) => return Poll::Ready(Err(error)),
4601            };
4602            let arguments = normalize_arguments(args);
4603            let envelope = match encode_value_envelope(&arguments, &state.payload_codec) {
4604                Ok(envelope) => envelope,
4605                Err(error) => return Poll::Ready(Err(error)),
4606            };
4607
4608            let mut command = serde_json::Map::from_iter([
4609                ("type".to_string(), json!("schedule_activity")),
4610                (
4611                    "activity_type".to_string(),
4612                    json!(self.activity_type.clone()),
4613                ),
4614                ("queue".to_string(), json!(task_queue)),
4615                ("arguments".to_string(), envelope),
4616            ]);
4617            for (field, value) in [
4618                ("start_to_close_timeout", options.start_to_close_timeout),
4619                (
4620                    "schedule_to_start_timeout",
4621                    options.schedule_to_start_timeout,
4622                ),
4623                (
4624                    "schedule_to_close_timeout",
4625                    options.schedule_to_close_timeout,
4626                ),
4627                ("heartbeat_timeout", options.heartbeat_timeout),
4628            ] {
4629                if let Some(value) = value {
4630                    command.insert(field.to_string(), json!(value));
4631                }
4632            }
4633            if let Some(retry_policy) = options.retry_policy {
4634                command.insert("retry_policy".to_string(), retry_policy);
4635            }
4636            state.commands.push(Value::Object(command));
4637            self.scheduled = true;
4638        }
4639
4640        Poll::Pending
4641    }
4642}
4643
4644/// Future returned by [`WorkflowContext::sleep`].
4645pub struct TimerCall {
4646    ctx: WorkflowContext,
4647    delay_seconds: Option<u64>,
4648    scheduled: bool,
4649    matched_pending: bool,
4650}
4651
4652impl Future for TimerCall {
4653    type Output = Result<()>;
4654
4655    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4656        if self.matched_pending {
4657            return Poll::Pending;
4658        }
4659
4660        let ctx = self.ctx.clone();
4661        let Some(requested_delay) = self.delay_seconds else {
4662            return Poll::Ready(Err(Error::TimerDurationOverflow));
4663        };
4664        let mut state = match ctx.state.lock() {
4665            Ok(state) => state,
4666            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4667        };
4668
4669        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4670            match recorded {
4671                RecordedCommand::Timer {
4672                    sequence,
4673                    delay_seconds,
4674                    fired,
4675                    ..
4676                } => {
4677                    if delay_seconds != requested_delay {
4678                        return Poll::Ready(Err(Error::NonDeterministicReplay(
4679                            ReplayFailure::new(
4680                                "timer_delay_mismatch",
4681                                Some(sequence),
4682                                Some(format!("timer:{delay_seconds}s")),
4683                                Some(format!("timer:{requested_delay}s")),
4684                                "recorded timer delay differs from the current workflow command",
4685                            ),
4686                        )));
4687                    }
4688                    state.command_cursor += 1;
4689                    if fired {
4690                        return Poll::Ready(Ok(()));
4691                    }
4692                    state.matched_recorded_pending = true;
4693                    self.scheduled = true;
4694                    self.matched_pending = true;
4695                    return Poll::Pending;
4696                }
4697                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
4698            }
4699        }
4700
4701        if !self.scheduled {
4702            state.commands.push(json!({
4703                "type": "start_timer",
4704                "delay_seconds": requested_delay,
4705            }));
4706            self.scheduled = true;
4707        }
4708
4709        Poll::Pending
4710    }
4711}
4712
4713/// Future returned by [`WorkflowContext::start_child_workflow`].
4714pub struct ChildWorkflowCall {
4715    ctx: WorkflowContext,
4716    workflow_type: String,
4717    options: ChildWorkflowOptions,
4718    args: Option<Result<Value>>,
4719    scheduled: bool,
4720    matched_pending: bool,
4721}
4722
4723impl Future for ChildWorkflowCall {
4724    type Output = Result<ChildWorkflowResult>;
4725
4726    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4727        if self.matched_pending {
4728            return Poll::Pending;
4729        }
4730
4731        let ctx = self.ctx.clone();
4732        let mut state = match ctx.state.lock() {
4733            Ok(state) => state,
4734            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4735        };
4736
4737        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4738            let sequence = recorded.sequence();
4739            match recorded {
4740                RecordedCommand::ChildWorkflow {
4741                    workflow_type,
4742                    outcome,
4743                    ..
4744                } => {
4745                    if let Some(recorded_type) = workflow_type {
4746                        if recorded_type != self.workflow_type {
4747                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4748                                ReplayFailure::new(
4749                                    "recorded_command_detail_mismatch",
4750                                    Some(sequence),
4751                                    Some(format!("child workflow:{recorded_type}")),
4752                                    Some(format!("child workflow:{}", self.workflow_type)),
4753                                    "recorded child workflow type differs from the current workflow command",
4754                                ),
4755                            )));
4756                        }
4757                    }
4758                    state.command_cursor += 1;
4759                    if let Some(outcome) = outcome {
4760                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
4761                    }
4762                    state.matched_recorded_pending = true;
4763                    self.scheduled = true;
4764                    self.matched_pending = true;
4765                    return Poll::Pending;
4766                }
4767                other => {
4768                    return Poll::Ready(Err(command_mismatch(
4769                        &other,
4770                        format!("child workflow:{}", self.workflow_type),
4771                    )));
4772                }
4773            }
4774        }
4775
4776        if !self.scheduled {
4777            if self.options.task_queue.trim().is_empty() {
4778                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4779                    "task_queue must not be empty".to_string(),
4780                )));
4781            }
4782            for (name, value) in [
4783                (
4784                    "execution_timeout_seconds",
4785                    self.options.execution_timeout_seconds,
4786                ),
4787                ("run_timeout_seconds", self.options.run_timeout_seconds),
4788            ] {
4789                if value == Some(0) {
4790                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
4791                        "{name} must be at least 1"
4792                    ))));
4793                }
4794            }
4795
4796            let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4797                Ok(args) => args,
4798                Err(error) => return Poll::Ready(Err(error)),
4799            };
4800            let arguments =
4801                match encode_value_envelope(&normalize_arguments(args), &state.payload_codec) {
4802                    Ok(arguments) => arguments,
4803                    Err(error) => return Poll::Ready(Err(error)),
4804                };
4805            let mut command = json!({
4806                "type": "start_child_workflow",
4807                "workflow_type": self.workflow_type,
4808                "queue": self.options.task_queue,
4809                "parent_close_policy": self.options.parent_close_policy.as_str(),
4810                "arguments": arguments,
4811            });
4812            let object = command
4813                .as_object_mut()
4814                .expect("child workflow command is always an object");
4815            if let Some(policy) = &self.options.retry_policy {
4816                let mut retry_policy = serde_json::Map::new();
4817                if let Some(max_attempts) = policy.max_attempts {
4818                    if max_attempts == 0 {
4819                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4820                            "retry_policy.max_attempts must be at least 1".to_string(),
4821                        )));
4822                    }
4823                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
4824                }
4825                if !policy.backoff_seconds.is_empty() {
4826                    retry_policy
4827                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
4828                }
4829                if !policy.non_retryable_error_types.is_empty() {
4830                    retry_policy.insert(
4831                        "non_retryable_error_types".to_string(),
4832                        json!(policy.non_retryable_error_types),
4833                    );
4834                }
4835                if retry_policy.is_empty() {
4836                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4837                        "retry_policy must configure at least one field".to_string(),
4838                    )));
4839                }
4840                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
4841            }
4842            if let Some(seconds) = self.options.execution_timeout_seconds {
4843                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
4844            }
4845            if let Some(seconds) = self.options.run_timeout_seconds {
4846                object.insert("run_timeout_seconds".to_string(), json!(seconds));
4847            }
4848            state.commands.push(command);
4849            self.scheduled = true;
4850        }
4851
4852        Poll::Pending
4853    }
4854}
4855
4856fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
4857    Error::NonDeterministicReplay(ReplayFailure::new(
4858        "recorded_command_mismatch",
4859        Some(recorded.sequence()),
4860        Some(recorded.shape().to_string()),
4861        Some(actual.into()),
4862        "current workflow command does not match the recorded durable command sequence",
4863    ))
4864}
4865
4866pub struct SignalCall {
4867    ctx: WorkflowContext,
4868    signal_name: String,
4869    opened_wait: bool,
4870    matched_pending: bool,
4871}
4872
4873impl Future for SignalCall {
4874    type Output = Result<Vec<Value>>;
4875
4876    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4877        if self.matched_pending {
4878            return Poll::Pending;
4879        }
4880
4881        let ctx = self.ctx.clone();
4882        let mut state = match ctx.state.lock() {
4883            Ok(state) => state,
4884            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4885        };
4886
4887        let signals = match signal_values(
4888            &state.history,
4889            &self.signal_name,
4890            &state.payload_codec,
4891            state.resume_signal.as_ref(),
4892        ) {
4893            Ok(signals) => signals,
4894            Err(error) => return Poll::Ready(Err(error)),
4895        };
4896        let cursor = *state.signal_cursors.get(&self.signal_name).unwrap_or(&0);
4897
4898        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4899            match recorded {
4900                RecordedCommand::SignalWait {
4901                    sequence,
4902                    signal_name,
4903                } => {
4904                    if let Some(recorded_name) = signal_name {
4905                        if recorded_name != self.signal_name {
4906                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4907                                ReplayFailure::new(
4908                                    "recorded_command_detail_mismatch",
4909                                    Some(sequence),
4910                                    Some(format!("signal wait:{recorded_name}")),
4911                                    Some(format!("signal wait:{}", self.signal_name)),
4912                                    "recorded signal name differs from the current workflow command",
4913                                ),
4914                            )));
4915                        }
4916                    }
4917
4918                    state.command_cursor += 1;
4919                    if cursor < signals.len() {
4920                        state
4921                            .signal_cursors
4922                            .insert(self.signal_name.clone(), cursor + 1);
4923                        return Poll::Ready(Ok(signals[cursor].clone()));
4924                    }
4925
4926                    state.matched_recorded_pending = true;
4927                    self.opened_wait = true;
4928                    self.matched_pending = true;
4929                    return Poll::Pending;
4930                }
4931                other => {
4932                    return Poll::Ready(Err(command_mismatch(
4933                        &other,
4934                        format!("signal wait:{}", self.signal_name),
4935                    )));
4936                }
4937            }
4938        }
4939
4940        if cursor < signals.len() {
4941            state
4942                .signal_cursors
4943                .insert(self.signal_name.clone(), cursor + 1);
4944            return Poll::Ready(Ok(signals[cursor].clone()));
4945        }
4946
4947        if !self.opened_wait {
4948            state.commands.push(json!({
4949                "type": "open_condition_wait",
4950                "condition_key": format!("signal:{}", self.signal_name)
4951            }));
4952            self.opened_wait = true;
4953        }
4954
4955        Poll::Pending
4956    }
4957}
4958
4959#[derive(Clone, Debug)]
4960pub struct ActivityContext {
4961    client: Client,
4962    pub task_id: String,
4963    pub activity_attempt_id: String,
4964    pub lease_owner: String,
4965    pub activity_type: String,
4966    pub attempt_number: u64,
4967    pub task_queue: String,
4968    pub worker_id: String,
4969}
4970
4971impl ActivityContext {
4972    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
4973        self.client
4974            .heartbeat_activity_task(
4975                &self.task_id,
4976                &self.activity_attempt_id,
4977                &self.lease_owner,
4978                serde_json::to_value(details)?,
4979            )
4980            .await
4981    }
4982}
4983
4984fn decode_task_arguments(value: Option<&Value>, codec: &str) -> Result<Value> {
4985    match value {
4986        Some(value) => Ok(normalize_arguments(decode_wire_value(value, codec)?)),
4987        None => Ok(Value::Array(Vec::new())),
4988    }
4989}
4990
4991fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
4992    let Some(signal_name) = task
4993        .signal_name
4994        .as_deref()
4995        .filter(|value| !value.is_empty())
4996    else {
4997        return Ok(None);
4998    };
4999    let Some(arguments) = task.signal_arguments.as_ref() else {
5000        return Ok(None);
5001    };
5002
5003    let decoded = normalize_arguments(decode_wire_value(arguments, &task.payload_codec)?);
5004    let Value::Array(arguments) = decoded else {
5005        unreachable!("normalize_arguments always returns an array");
5006    };
5007
5008    Ok(Some(ResumeSignal {
5009        signal_id: task.workflow_signal_id.clone(),
5010        signal_name: signal_name.to_string(),
5011        arguments,
5012    }))
5013}
5014
5015fn normalize_arguments(value: Value) -> Value {
5016    match value {
5017        Value::Null => Value::Array(Vec::new()),
5018        Value::Array(_) => value,
5019        other => Value::Array(vec![other]),
5020    }
5021}
5022
5023fn recorded_commands(
5024    events: &[HistoryEvent],
5025    fallback_codec: &str,
5026    parent: WorkflowIdentity,
5027) -> Result<Vec<RecordedCommand>> {
5028    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
5029
5030    for event in events {
5031        let is_activity = matches!(
5032            event.event_type.as_str(),
5033            "ActivityScheduled"
5034                | "ActivityStarted"
5035                | "ActivityHeartbeatRecorded"
5036                | "ActivityRetryScheduled"
5037                | "ActivityCompleted"
5038                | "ActivityFailed"
5039                | "ActivityCancelled"
5040                | "ActivityTimedOut"
5041        );
5042        let is_workflow_timer = matches!(
5043            event.event_type.as_str(),
5044            "TimerScheduled" | "TimerCancelled" | "TimerFired"
5045        ) && !is_internal_timer_event(event);
5046        let is_child_workflow = matches!(
5047            event.event_type.as_str(),
5048            "ChildWorkflowScheduled"
5049                | "ChildRunCompleted"
5050                | "ChildRunFailed"
5051                | "ChildRunCancelled"
5052                | "ChildRunTerminated"
5053        );
5054        let is_signal_wait = is_recorded_signal_wait_event(event);
5055        if !is_activity && !is_workflow_timer && !is_child_workflow && !is_signal_wait {
5056            continue;
5057        }
5058
5059        let sequence = durable_event_sequence(event).ok_or_else(|| {
5060            Error::NonDeterministicReplay(ReplayFailure::new(
5061                "durable_command_sequence_missing",
5062                None,
5063                Some("positive workflow sequence".to_string()),
5064                Some(event.event_type.clone()),
5065                "durable command history event has no workflow sequence",
5066            ))
5067        })?;
5068        if sequence == 0 {
5069            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5070                "durable_command_sequence_invalid",
5071                Some(sequence),
5072                Some("positive workflow sequence".to_string()),
5073                Some(sequence.to_string()),
5074                "durable command history uses an invalid workflow sequence",
5075            )));
5076        }
5077        events_by_sequence.entry(sequence).or_default().push(event);
5078    }
5079
5080    events_by_sequence
5081        .into_iter()
5082        .map(|(sequence, sequence_events)| {
5083            let activity_events: Vec<_> = sequence_events
5084                .iter()
5085                .copied()
5086                .filter(|event| event.event_type.starts_with("Activity"))
5087                .collect();
5088            let timer_events: Vec<_> = sequence_events
5089                .iter()
5090                .copied()
5091                .filter(|event| event.event_type.starts_with("Timer"))
5092                .collect();
5093            let child_events: Vec<_> = sequence_events
5094                .iter()
5095                .copied()
5096                .filter(|event| {
5097                    event.event_type == "ChildWorkflowScheduled"
5098                        || event.event_type.starts_with("ChildRun")
5099                })
5100                .collect();
5101            let signal_wait_events: Vec<_> = sequence_events
5102                .iter()
5103                .copied()
5104                .filter(|event| is_recorded_signal_wait_event(event))
5105                .collect();
5106
5107            let command_kind_count = usize::from(!activity_events.is_empty())
5108                + usize::from(!timer_events.is_empty())
5109                + usize::from(!child_events.is_empty())
5110                + usize::from(!signal_wait_events.is_empty());
5111            if command_kind_count > 1 {
5112                let actual = [
5113                    (!activity_events.is_empty()).then_some("activity"),
5114                    (!timer_events.is_empty()).then_some("timer"),
5115                    (!child_events.is_empty()).then_some("child workflow"),
5116                    (!signal_wait_events.is_empty()).then_some("signal wait"),
5117                ]
5118                .into_iter()
5119                .flatten()
5120                .collect::<Vec<_>>()
5121                .join(" and ");
5122                return Err(invalid_recorded_history(
5123                    "durable_command_sequence_collision",
5124                    sequence,
5125                    "one durable command kind",
5126                    &actual,
5127                    "one workflow sequence records more than one durable command kind",
5128                ));
5129            }
5130
5131            if !activity_events.is_empty() {
5132                let scheduled_count = activity_events
5133                    .iter()
5134                    .filter(|event| event.event_type == "ActivityScheduled")
5135                    .count();
5136                if scheduled_count > 1 {
5137                    return Err(invalid_recorded_history(
5138                        "duplicate_activity_schedule",
5139                        sequence,
5140                        "at most one ActivityScheduled event",
5141                        "multiple ActivityScheduled events",
5142                        "activity history schedules more than one command at one workflow sequence",
5143                    ));
5144                }
5145                let activity_type = activity_events.iter().find_map(|event| {
5146                    event
5147                        .payload
5148                        .get("activity_type")
5149                        .or_else(|| event.payload.get("activity_name"))
5150                        .and_then(Value::as_str)
5151                        .map(str::to_string)
5152                });
5153                if activity_events.iter().filter_map(|event| {
5154                    event
5155                        .payload
5156                        .get("activity_type")
5157                        .or_else(|| event.payload.get("activity_name"))
5158                        .and_then(Value::as_str)
5159                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
5160                    return Err(invalid_recorded_history(
5161                        "activity_identity_mismatch",
5162                        sequence,
5163                        activity_type.as_deref().unwrap_or("one activity identity"),
5164                        "conflicting activity identities",
5165                        "activity lifecycle events at one workflow sequence disagree on identity",
5166                    ));
5167                }
5168                let terminal: Vec<_> = activity_events
5169                    .iter()
5170                    .copied()
5171                    .filter(|event| {
5172                        matches!(
5173                            event.event_type.as_str(),
5174                            "ActivityCompleted"
5175                                | "ActivityFailed"
5176                                | "ActivityCancelled"
5177                                | "ActivityTimedOut"
5178                        )
5179                    })
5180                    .collect();
5181                if terminal.len() > 1 {
5182                    return Err(invalid_recorded_history(
5183                        "duplicate_activity_terminal_event",
5184                        sequence,
5185                        "at most one terminal activity event",
5186                        "multiple terminal activity events",
5187                        "activity history settles one command more than once",
5188                    ));
5189                }
5190                let outcome = terminal
5191                    .first()
5192                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
5193                    .transpose()?;
5194                let options = activity_events
5195                    .iter()
5196                    .find(|event| event.event_type == "ActivityScheduled")
5197                    .and_then(|event| event.payload.get("activity"))
5198                    .and_then(Value::as_object)
5199                    .map(|activity| RecordedActivityOptions {
5200                        task_queue: recorded_optional_string(activity, "queue"),
5201                        execution_mode: recorded_optional_string(activity, "execution_mode"),
5202                        retry_policy: recorded_activity_retry_snapshot(
5203                            activity.get("retry_policy"),
5204                        ),
5205                    });
5206                return Ok(RecordedCommand::Activity {
5207                    sequence,
5208                    activity_type,
5209                    options,
5210                    outcome,
5211                });
5212            }
5213
5214            if !child_events.is_empty() {
5215                let scheduled: Vec<_> = child_events
5216                    .iter()
5217                    .copied()
5218                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
5219                    .collect();
5220                if scheduled.len() != 1 {
5221                    return Err(invalid_recorded_history(
5222                        "child_workflow_schedule_missing_or_duplicate",
5223                        sequence,
5224                        "one ChildWorkflowScheduled event",
5225                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
5226                        "child workflow replay requires exactly one recorded schedule event",
5227                    ));
5228                }
5229                let workflow_type = child_events.iter().find_map(|event| {
5230                    event
5231                        .payload
5232                        .get("child_workflow_type")
5233                        .or_else(|| event.payload.get("workflow_type"))
5234                        .and_then(Value::as_str)
5235                        .filter(|value| !value.is_empty())
5236                        .map(str::to_string)
5237                });
5238                if child_events
5239                    .iter()
5240                    .filter_map(|event| {
5241                        event
5242                            .payload
5243                            .get("child_workflow_type")
5244                            .or_else(|| event.payload.get("workflow_type"))
5245                            .and_then(Value::as_str)
5246                    })
5247                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
5248                {
5249                    return Err(invalid_recorded_history(
5250                        "child_workflow_identity_mismatch",
5251                        sequence,
5252                        workflow_type
5253                            .as_deref()
5254                            .unwrap_or("one child workflow type"),
5255                        "conflicting child workflow types",
5256                        "child workflow lifecycle events at one sequence disagree on type",
5257                    ));
5258                }
5259                let mut outcomes = child_workflow_outcomes(
5260                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
5261                    fallback_codec,
5262                    parent.clone(),
5263                )?;
5264                if outcomes.len() > 1 {
5265                    return Err(invalid_recorded_history(
5266                        "duplicate_child_workflow_terminal_event",
5267                        sequence,
5268                        "at most one terminal child event",
5269                        "multiple terminal child events",
5270                        "child workflow history settles one command more than once",
5271                    ));
5272                }
5273                return Ok(RecordedCommand::ChildWorkflow {
5274                    sequence,
5275                    workflow_type,
5276                    outcome: outcomes.pop(),
5277                });
5278            }
5279
5280            if !signal_wait_events.is_empty() {
5281                let opened_count = signal_wait_events
5282                    .iter()
5283                    .filter(|event| {
5284                        matches!(
5285                            event.event_type.as_str(),
5286                            "SignalWaitOpened" | "ConditionWaitOpened"
5287                        )
5288                    })
5289                    .count();
5290                if opened_count > 1 {
5291                    return Err(invalid_recorded_history(
5292                        "duplicate_signal_wait_open",
5293                        sequence,
5294                        "at most one signal wait open event",
5295                        "multiple signal wait open events",
5296                        "signal history opens more than one durable wait at one workflow sequence",
5297                    ));
5298                }
5299
5300                let signal_name = signal_wait_events
5301                    .iter()
5302                    .find_map(|event| recorded_signal_wait_name(event));
5303                if signal_wait_events
5304                    .iter()
5305                    .filter_map(|event| recorded_signal_wait_name(event))
5306                    .any(|candidate| Some(candidate.as_str()) != signal_name.as_deref())
5307                {
5308                    return Err(invalid_recorded_history(
5309                        "signal_wait_identity_mismatch",
5310                        sequence,
5311                        signal_name.as_deref().unwrap_or("one signal name"),
5312                        "conflicting signal names",
5313                        "signal wait lifecycle events at one workflow sequence disagree on identity",
5314                    ));
5315                }
5316                return Ok(RecordedCommand::SignalWait {
5317                    sequence,
5318                    signal_name,
5319                });
5320            }
5321
5322            let scheduled: Vec<_> = timer_events
5323                .iter()
5324                .copied()
5325                .filter(|event| event.event_type == "TimerScheduled")
5326                .collect();
5327            let fired: Vec<_> = timer_events
5328                .iter()
5329                .copied()
5330                .filter(|event| event.event_type == "TimerFired")
5331                .collect();
5332            if scheduled.len() != 1 {
5333                return Err(invalid_recorded_history(
5334                    "timer_schedule_missing_or_duplicate",
5335                    sequence,
5336                    "one TimerScheduled event",
5337                    &format!("{} TimerScheduled events", scheduled.len()),
5338                    "timer replay requires exactly one recorded schedule event",
5339                ));
5340            }
5341            if fired.len() > 1 {
5342                return Err(invalid_recorded_history(
5343                    "duplicate_timer_fire",
5344                    sequence,
5345                    "at most one TimerFired event",
5346                    "multiple TimerFired events",
5347                    "timer history contains more than one fire event for a workflow sequence",
5348                ));
5349            }
5350
5351            let scheduled = scheduled[0];
5352            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
5353            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
5354            if let Some(fired) = fired.first() {
5355                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
5356                if fired_timer_id != timer_id {
5357                    return Err(invalid_recorded_history(
5358                        "timer_identity_mismatch",
5359                        sequence,
5360                        &timer_id,
5361                        &fired_timer_id,
5362                        "TimerFired does not correspond to the recorded TimerScheduled event",
5363                    ));
5364                }
5365                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
5366                if fired_delay != delay_seconds {
5367                    return Err(invalid_recorded_history(
5368                        "timer_history_delay_mismatch",
5369                        sequence,
5370                        &delay_seconds.to_string(),
5371                        &fired_delay.to_string(),
5372                        "TimerScheduled and TimerFired record different delays",
5373                    ));
5374                }
5375            }
5376
5377            Ok(RecordedCommand::Timer {
5378                sequence,
5379                delay_seconds,
5380                fired: !fired.is_empty(),
5381            })
5382        })
5383        .collect()
5384}
5385
5386fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
5387    event
5388        .payload
5389        .get("sequence")
5390        .or_else(|| event.payload.get("workflow_sequence"))
5391        .or_else(|| event.raw.get("sequence"))
5392        .or_else(|| event.raw.get("workflow_sequence"))
5393        .and_then(value_as_u64)
5394}
5395
5396fn is_internal_timer_event(event: &HistoryEvent) -> bool {
5397    matches!(
5398        event
5399            .payload
5400            .get("timer_kind")
5401            .or_else(|| event.raw.get("timer_kind"))
5402            .and_then(Value::as_str),
5403        Some("condition_timeout" | "signal_timeout")
5404    )
5405}
5406
5407fn recorded_signal_wait_name(event: &HistoryEvent) -> Option<String> {
5408    match event.event_type.as_str() {
5409        "SignalWaitOpened" | "SignalApplied" => event
5410            .payload
5411            .get("signal_name")
5412            .or_else(|| event.raw.get("signal_name"))
5413            .and_then(Value::as_str)
5414            .filter(|value| !value.is_empty())
5415            .map(str::to_string),
5416        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5417            .payload
5418            .get("condition_key")
5419            .or_else(|| event.raw.get("condition_key"))
5420            .and_then(Value::as_str)
5421            .and_then(|key| key.strip_prefix("signal:"))
5422            .filter(|value| !value.is_empty())
5423            .map(str::to_string),
5424        _ => None,
5425    }
5426}
5427
5428fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
5429    match event.event_type.as_str() {
5430        "SignalWaitOpened" | "SignalApplied" => true,
5431        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5432            .payload
5433            .get("condition_key")
5434            .or_else(|| event.raw.get("condition_key"))
5435            .and_then(Value::as_str)
5436            .is_some_and(|key| key.starts_with("signal:")),
5437        _ => false,
5438    }
5439}
5440
5441fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
5442    event
5443        .payload
5444        .get(field)
5445        .and_then(Value::as_str)
5446        .filter(|value| !value.is_empty())
5447        .map(str::to_string)
5448        .ok_or_else(|| {
5449            invalid_recorded_history(
5450                "timer_history_field_missing",
5451                sequence,
5452                field,
5453                &event.event_type,
5454                "timer history is missing a required identity field",
5455            )
5456        })
5457}
5458
5459fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
5460    event
5461        .payload
5462        .get(field)
5463        .and_then(value_as_u64)
5464        .ok_or_else(|| {
5465            invalid_recorded_history(
5466                "timer_history_field_missing",
5467                sequence,
5468                field,
5469                &event.event_type,
5470                "timer history is missing a required numeric field",
5471            )
5472        })
5473}
5474
5475fn invalid_recorded_history(
5476    reason: &str,
5477    sequence: u64,
5478    expected: &str,
5479    actual: &str,
5480    message: &str,
5481) -> Error {
5482    Error::NonDeterministicReplay(ReplayFailure::new(
5483        reason,
5484        Some(sequence),
5485        Some(expected.to_string()),
5486        Some(actual.to_string()),
5487        message,
5488    ))
5489}
5490
5491type ActivityOutcome = std::result::Result<Value, ActivityFailure>;
5492
5493fn activity_outcome(
5494    event: &HistoryEvent,
5495    fallback_codec: &str,
5496    recorded_activity_type: Option<String>,
5497) -> Result<ActivityOutcome> {
5498    if event.event_type == "ActivityCompleted" {
5499        let codec = event
5500            .payload
5501            .get("payload_codec")
5502            .and_then(Value::as_str)
5503            .unwrap_or(fallback_codec);
5504        return Ok(Ok(decode_wire_value(
5505            event.payload.get("result").unwrap_or(&Value::Null),
5506            codec,
5507        )?));
5508    }
5509
5510    let payload = &event.payload;
5511    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
5512        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
5513        "ActivityCancelled" => (
5514            ActivityFailureKind::Cancelled,
5515            "cancelled",
5516            "activity was cancelled",
5517        ),
5518        "ActivityTimedOut" => (
5519            ActivityFailureKind::TimedOut,
5520            "timeout",
5521            "activity timed out",
5522        ),
5523        _ => unreachable!("activity_outcome is called only for terminal activity events"),
5524    };
5525    let exception = payload
5526        .get("exception")
5527        .filter(|value| !value.is_null())
5528        .cloned();
5529    let failure_category = payload_string(payload, "failure_category");
5530    let timeout_kind = payload_string(payload, "timeout_kind");
5531    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
5532        ActivityFailureKind::Failed => failure_category
5533            .clone()
5534            .unwrap_or_else(|| fallback_reason.to_string()),
5535        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
5536        ActivityFailureKind::TimedOut => timeout_kind
5537            .clone()
5538            .unwrap_or_else(|| fallback_reason.to_string()),
5539    });
5540    let message = payload_string(payload, "message")
5541        .or_else(|| {
5542            exception
5543                .as_ref()
5544                .and_then(|value| payload_string(value, "message"))
5545        })
5546        .unwrap_or_else(|| fallback_message.to_string());
5547
5548    Ok(Err(ActivityFailure {
5549        kind,
5550        reason,
5551        message,
5552        activity_execution_id: payload_string(payload, "activity_execution_id"),
5553        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
5554        activity_type: payload_string(payload, "activity_type")
5555            .or_else(|| payload_string(payload, "activity_name"))
5556            .or(recorded_activity_type),
5557        activity_class: payload_string(payload, "activity_class"),
5558        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
5559        failure_id: payload_string(payload, "failure_id"),
5560        failure_category,
5561        timeout_kind,
5562        non_retryable: payload
5563            .get("non_retryable")
5564            .and_then(Value::as_bool)
5565            .unwrap_or(false),
5566        exception_type: payload_string(payload, "exception_type").or_else(|| {
5567            exception
5568                .as_ref()
5569                .and_then(|value| payload_string(value, "type"))
5570        }),
5571        exception_class: payload_string(payload, "exception_class").or_else(|| {
5572            exception
5573                .as_ref()
5574                .and_then(|value| payload_string(value, "class"))
5575        }),
5576        code: payload
5577            .get("code")
5578            .filter(|value| !value.is_null())
5579            .cloned(),
5580        exception,
5581    }))
5582}
5583
5584type ChildWorkflowOutcome = std::result::Result<ChildWorkflowResult, ChildWorkflowFailure>;
5585
5586fn child_workflow_outcomes(
5587    events: &[HistoryEvent],
5588    fallback_codec: &str,
5589    parent: WorkflowIdentity,
5590) -> Result<Vec<ChildWorkflowOutcome>> {
5591    let mut outcomes = Vec::new();
5592
5593    for event in events {
5594        let kind = match event.event_type.as_str() {
5595            "ChildRunCompleted" => None,
5596            "ChildRunFailed" => Some((
5597                ChildWorkflowFailureKind::Failed,
5598                "child_workflow",
5599                "child workflow failed",
5600            )),
5601            "ChildRunCancelled" => Some((
5602                ChildWorkflowFailureKind::Cancelled,
5603                "cancelled",
5604                "child workflow was cancelled",
5605            )),
5606            "ChildRunTerminated" => Some((
5607                ChildWorkflowFailureKind::Terminated,
5608                "terminated",
5609                "child workflow was terminated",
5610            )),
5611            _ => continue,
5612        };
5613        let payload = &event.payload;
5614        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
5615        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
5616        let child_workflow_type = payload_string(payload, "child_workflow_type");
5617
5618        if let Some((kind, reason, fallback_message)) = kind {
5619            let exception = payload
5620                .get("exception")
5621                .filter(|value| !value.is_null())
5622                .cloned();
5623            let message = payload_string(payload, "message")
5624                .or_else(|| {
5625                    exception
5626                        .as_ref()
5627                        .and_then(|value| payload_string(value, "message"))
5628                })
5629                .unwrap_or_else(|| fallback_message.to_string());
5630            let exception_type = payload_string(payload, "exception_type").or_else(|| {
5631                exception
5632                    .as_ref()
5633                    .and_then(|value| payload_string(value, "type"))
5634            });
5635            let exception_class = payload_string(payload, "exception_class").or_else(|| {
5636                exception
5637                    .as_ref()
5638                    .and_then(|value| payload_string(value, "class"))
5639            });
5640            outcomes.push(Err(ChildWorkflowFailure {
5641                kind,
5642                reason: reason.to_string(),
5643                message,
5644                parent_workflow_id: parent.workflow_id.clone(),
5645                parent_workflow_run_id: parent.run_id.clone(),
5646                child_workflow_id,
5647                child_workflow_run_id,
5648                child_workflow_type,
5649                failure_id: payload_string(payload, "failure_id"),
5650                failure_category: payload_string(payload, "failure_category"),
5651                exception_type,
5652                exception_class,
5653                non_retryable: payload
5654                    .get("non_retryable")
5655                    .and_then(Value::as_bool)
5656                    .unwrap_or(false),
5657                code: payload
5658                    .get("code")
5659                    .filter(|value| !value.is_null())
5660                    .cloned(),
5661                exception,
5662            }));
5663            continue;
5664        }
5665
5666        let codec = payload
5667            .get("payload_codec")
5668            .and_then(Value::as_str)
5669            .unwrap_or(fallback_codec);
5670        let result = payload
5671            .get("result")
5672            .or_else(|| payload.get("output"))
5673            .unwrap_or(&Value::Null);
5674        outcomes.push(Ok(ChildWorkflowResult {
5675            parent: parent.clone(),
5676            child: WorkflowIdentity {
5677                workflow_id: child_workflow_id,
5678                run_id: child_workflow_run_id,
5679            },
5680            child_workflow_type,
5681            result: decode_wire_value(result, codec)?,
5682        }));
5683    }
5684
5685    Ok(outcomes)
5686}
5687
5688fn payload_string(payload: &Value, key: &str) -> Option<String> {
5689    payload
5690        .get(key)
5691        .and_then(Value::as_str)
5692        .filter(|value| !value.is_empty())
5693        .map(str::to_string)
5694}
5695
5696fn workflow_failure_command(error: &Error) -> Value {
5697    let (exception_type, exception_class, properties) = match error {
5698        Error::ActivityFailed(failure) => (
5699            match failure.kind {
5700                ActivityFailureKind::Failed => "ActivityFailed",
5701                ActivityFailureKind::Cancelled => "ActivityCancelled",
5702                ActivityFailureKind::TimedOut => "ActivityTimedOut",
5703            },
5704            "durable_workflow::ActivityFailure",
5705            json!({
5706                "reason": failure.reason,
5707                "activity_execution_id": failure.activity_execution_id,
5708                "activity_attempt_id": failure.activity_attempt_id,
5709                "activity_type": failure.activity_type,
5710                "activity_class": failure.activity_class,
5711                "attempt_number": failure.attempt_number,
5712                "failure_id": failure.failure_id,
5713                "failure_category": failure.failure_category,
5714                "timeout_kind": failure.timeout_kind,
5715                "activity_non_retryable": failure.non_retryable,
5716                "activity_exception_type": failure.exception_type,
5717                "activity_exception_class": failure.exception_class,
5718                "activity_code": failure.code,
5719                "activity_exception": failure.exception,
5720            }),
5721        ),
5722        Error::ChildWorkflowFailed(failure) => (
5723            match failure.kind {
5724                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
5725                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
5726                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
5727            },
5728            "durable_workflow::ChildWorkflowFailure",
5729            json!({
5730                "reason": failure.reason,
5731                "parent_workflow_id": failure.parent_workflow_id,
5732                "parent_workflow_run_id": failure.parent_workflow_run_id,
5733                "child_workflow_id": failure.child_workflow_id,
5734                "child_workflow_run_id": failure.child_workflow_run_id,
5735                "child_workflow_type": failure.child_workflow_type,
5736                "failure_id": failure.failure_id,
5737                "failure_category": failure.failure_category,
5738                "child_exception_type": failure.exception_type,
5739                "child_exception_class": failure.exception_class,
5740                "child_non_retryable": failure.non_retryable,
5741                "child_code": failure.code,
5742                "child_exception": failure.exception,
5743            }),
5744        ),
5745        Error::NonDeterministicReplay(_) => (
5746            "NonDeterministicReplay",
5747            "durable_workflow::Error",
5748            Value::Null,
5749        ),
5750        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
5751    };
5752    let non_retryable = match error {
5753        Error::ActivityFailed(failure) => failure.non_retryable,
5754        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
5755        Error::NonDeterministicReplay(_) => true,
5756        _ => false,
5757    };
5758
5759    json!({
5760        "type": "fail_workflow",
5761        "message": error.to_string(),
5762        "exception_type": exception_type,
5763        "exception_class": exception_class,
5764        "non_retryable": non_retryable,
5765        "exception": {
5766            "type": exception_type,
5767            "class": exception_class,
5768            "message": error.to_string(),
5769            "properties": properties,
5770        }
5771    })
5772}
5773
5774fn signal_values(
5775    events: &[HistoryEvent],
5776    signal_name: &str,
5777    fallback_codec: &str,
5778    resume_signal: Option<&ResumeSignal>,
5779) -> Result<Vec<Vec<Value>>> {
5780    let mut signals = Vec::new();
5781
5782    for event in events {
5783        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
5784            continue;
5785        }
5786
5787        if event.payload.get("signal_name").and_then(Value::as_str) != Some(signal_name) {
5788            continue;
5789        }
5790
5791        let codec = event
5792            .payload
5793            .get("payload_codec")
5794            .and_then(Value::as_str)
5795            .unwrap_or(fallback_codec);
5796        let raw = event
5797            .payload
5798            .get("value")
5799            .or_else(|| event.payload.get("input"))
5800            .or_else(|| event.payload.get("arguments"));
5801        let decoded = match raw.filter(|value| !value.is_null()) {
5802            Some(value) => decode_wire_value(value, codec)?,
5803            None => resume_signal
5804                .filter(|signal| resume_signal_matches_event(signal, event, signal_name))
5805                .map(|signal| Value::Array(signal.arguments.clone()))
5806                .unwrap_or_else(|| Value::Array(Vec::new())),
5807        };
5808        let args = match normalize_arguments(decoded) {
5809            Value::Array(values) => values,
5810            _ => unreachable!("normalize_arguments always returns an array"),
5811        };
5812        signals.push(args);
5813    }
5814
5815    Ok(signals)
5816}
5817
5818fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5819    let Some(export_events) = task
5820        .history_export
5821        .as_ref()
5822        .and_then(|export| export.get("history_events"))
5823        .and_then(Value::as_array)
5824    else {
5825        return Ok(());
5826    };
5827
5828    if export_events.len() > task.history_events.len() {
5829        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
5830    }
5831
5832    Ok(())
5833}
5834
5835fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5836    let Some(export) = task.history_export.as_ref() else {
5837        return Ok(());
5838    };
5839    let signals = export
5840        .get("signals")
5841        .and_then(Value::as_array)
5842        .cloned()
5843        .unwrap_or_default();
5844    let activities = export
5845        .get("activities")
5846        .and_then(Value::as_array)
5847        .cloned()
5848        .unwrap_or_default();
5849    let export_codec = export
5850        .get("payloads")
5851        .and_then(|payloads| payloads.get("codec"))
5852        .and_then(Value::as_str)
5853        .unwrap_or(&task.payload_codec)
5854        .to_string();
5855    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
5856
5857    for event in &mut task.history_events {
5858        if event.event_type == "ActivityCompleted" {
5859            let sequence = event
5860                .payload
5861                .get("sequence")
5862                .or_else(|| event.payload.get("workflow_sequence"))
5863                .and_then(value_as_u64);
5864            let Some(activity) = sequence.and_then(|sequence| {
5865                activities.iter().find(|activity| {
5866                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
5867                })
5868            }) else {
5869                continue;
5870            };
5871            let Some(payload) = event.payload.as_object_mut() else {
5872                continue;
5873            };
5874            if missing_payload(payload.get("result")) {
5875                if let Some(result) = activity
5876                    .get("result")
5877                    .filter(|value| !missing_payload(Some(value)))
5878                {
5879                    payload.insert("result".to_string(), result.clone());
5880                }
5881            }
5882            for field in ["payload_codec", "activity_type"] {
5883                if payload
5884                    .get(field)
5885                    .and_then(Value::as_str)
5886                    .unwrap_or_default()
5887                    .is_empty()
5888                {
5889                    if let Some(value) = activity.get(field) {
5890                        payload.insert(field.to_string(), value.clone());
5891                    }
5892                }
5893            }
5894            continue;
5895        }
5896
5897        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
5898            continue;
5899        }
5900        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
5901        let command_id = event
5902            .payload
5903            .get("workflow_command_id")
5904            .or_else(|| event.raw.get("workflow_command_id"))
5905            .and_then(Value::as_str);
5906        let signal_name = event
5907            .payload
5908            .get("signal_name")
5909            .and_then(Value::as_str)
5910            .unwrap_or_default()
5911            .to_string();
5912        let matched = signals
5913            .iter()
5914            .find(|signal| {
5915                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
5916            })
5917            .or_else(|| {
5918                signals.iter().find(|signal| {
5919                    command_id.is_some()
5920                        && signal.get("command_id").and_then(Value::as_str) == command_id
5921                })
5922            })
5923            .or_else(|| {
5924                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
5925                let signal = signals
5926                    .iter()
5927                    .filter(|signal| {
5928                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
5929                    })
5930                    .nth(*offset);
5931                if signal.is_some() {
5932                    *offset += 1;
5933                }
5934                signal
5935            });
5936        let Some(signal) = matched else {
5937            continue;
5938        };
5939        let signal_codec = signal
5940            .get("payload_codec")
5941            .and_then(Value::as_str)
5942            .unwrap_or(&export_codec);
5943        let Some(payload) = event.payload.as_object_mut() else {
5944            continue;
5945        };
5946        if missing_payload(payload.get("arguments")) {
5947            if let Some(arguments) = signal
5948                .get("arguments")
5949                .filter(|value| !missing_payload(Some(value)))
5950            {
5951                let envelope = match arguments {
5952                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
5953                    other => other.clone(),
5954                };
5955                payload.insert("arguments".to_string(), envelope);
5956            }
5957        }
5958        if payload
5959            .get("payload_codec")
5960            .and_then(Value::as_str)
5961            .unwrap_or_default()
5962            .is_empty()
5963        {
5964            payload.insert("payload_codec".to_string(), json!(signal_codec));
5965        }
5966    }
5967
5968    Ok(())
5969}
5970
5971fn missing_payload(value: Option<&Value>) -> bool {
5972    match value {
5973        None | Some(Value::Null) => true,
5974        Some(Value::String(value)) => value.is_empty(),
5975        Some(_) => false,
5976    }
5977}
5978
5979fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
5980    let export_signals = task
5981        .history_export
5982        .as_ref()
5983        .and_then(|export| export.get("signals"))
5984        .and_then(Value::as_array)
5985        .cloned()
5986        .unwrap_or_default();
5987    let export_codec = task
5988        .history_export
5989        .as_ref()
5990        .and_then(|export| export.get("payloads"))
5991        .and_then(|payloads| payloads.get("codec"))
5992        .and_then(Value::as_str)
5993        .unwrap_or(&task.payload_codec);
5994    let mut name_offsets: HashMap<String, usize> = HashMap::new();
5995    let mut signals = Vec::new();
5996
5997    for event in &task.history_events {
5998        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
5999            continue;
6000        }
6001
6002        let name = event
6003            .payload
6004            .get("signal_name")
6005            .and_then(Value::as_str)
6006            .unwrap_or_default();
6007        if name.is_empty() {
6008            continue;
6009        }
6010        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
6011        let command_id = event
6012            .payload
6013            .get("workflow_command_id")
6014            .or_else(|| event.raw.get("workflow_command_id"))
6015            .and_then(Value::as_str);
6016        let matched_export = export_signals
6017            .iter()
6018            .find(|candidate| {
6019                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
6020            })
6021            .or_else(|| {
6022                export_signals.iter().find(|candidate| {
6023                    command_id.is_some()
6024                        && candidate.get("command_id").and_then(Value::as_str) == command_id
6025                })
6026            })
6027            .or_else(|| {
6028                let offset = name_offsets.entry(name.to_string()).or_default();
6029                let candidate = export_signals
6030                    .iter()
6031                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
6032                    .nth(*offset);
6033                if candidate.is_some() {
6034                    *offset += 1;
6035                }
6036                candidate
6037            });
6038        let codec = event
6039            .payload
6040            .get("payload_codec")
6041            .and_then(Value::as_str)
6042            .or_else(|| {
6043                matched_export
6044                    .and_then(|signal| signal.get("payload_codec"))
6045                    .and_then(Value::as_str)
6046            })
6047            .unwrap_or(export_codec);
6048        let raw_arguments = event
6049            .payload
6050            .get("value")
6051            .or_else(|| event.payload.get("input"))
6052            .or_else(|| event.payload.get("arguments"))
6053            .filter(|value| !value.is_null())
6054            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
6055        let arguments = decode_query_signal_arguments(raw_arguments, codec)?;
6056        let workflow_sequence = event
6057            .payload
6058            .get("workflow_sequence")
6059            .and_then(value_as_u64)
6060            .or_else(|| {
6061                matched_export
6062                    .and_then(|signal| signal.get("workflow_sequence"))
6063                    .and_then(value_as_u64)
6064            });
6065
6066        signals.push(QuerySignal {
6067            id: signal_id.map(str::to_string).or_else(|| {
6068                matched_export
6069                    .and_then(|signal| signal.get("id"))
6070                    .and_then(Value::as_str)
6071                    .map(str::to_string)
6072            }),
6073            name: name.to_string(),
6074            arguments,
6075            workflow_sequence,
6076        });
6077    }
6078
6079    if signals.is_empty() {
6080        for signal in export_signals {
6081            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
6082                continue;
6083            }
6084            let Some(name) = signal.get("name").and_then(Value::as_str) else {
6085                continue;
6086            };
6087            let codec = signal
6088                .get("payload_codec")
6089                .and_then(Value::as_str)
6090                .unwrap_or(export_codec);
6091            let arguments = decode_query_signal_arguments(signal.get("arguments"), codec)?;
6092            signals.push(QuerySignal {
6093                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
6094                name: name.to_string(),
6095                arguments,
6096                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
6097            });
6098        }
6099        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
6100    }
6101
6102    Ok(signals)
6103}
6104
6105fn decode_query_signal_arguments(raw: Option<&Value>, codec: &str) -> Result<Vec<Value>> {
6106    let decoded = match raw.filter(|value| !value.is_null()) {
6107        Some(value) => decode_wire_value(value, codec)?,
6108        None => Value::Array(Vec::new()),
6109    };
6110    let Value::Array(arguments) = normalize_arguments(decoded) else {
6111        unreachable!("normalize_arguments always returns an array");
6112    };
6113    Ok(arguments)
6114}
6115
6116fn value_as_u64(value: &Value) -> Option<u64> {
6117    value
6118        .as_u64()
6119        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
6120}
6121
6122fn resume_signal_matches_event(
6123    resume_signal: &ResumeSignal,
6124    event: &HistoryEvent,
6125    signal_name: &str,
6126) -> bool {
6127    if resume_signal.signal_name != signal_name {
6128        return false;
6129    }
6130
6131    match (
6132        resume_signal.signal_id.as_deref(),
6133        event.payload.get("signal_id").and_then(Value::as_str),
6134    ) {
6135        (Some(resume_id), Some(event_id)) => resume_id == event_id,
6136        _ => true,
6137    }
6138}
6139
6140#[cfg(test)]
6141mod tests {
6142    use super::*;
6143    use std::{
6144        io::{Read, Write},
6145        net::{SocketAddr, TcpListener, TcpStream},
6146        thread,
6147    };
6148
6149    #[derive(Clone, Debug, Default, PartialEq)]
6150    struct ReplayCounterState {
6151        loaded: Option<String>,
6152        count: i64,
6153        finished: bool,
6154    }
6155
6156    fn replay_counter_worker() -> Worker {
6157        let client = Client::new("http://127.0.0.1:8080").expect("client");
6158        let mut worker = Worker::new(client, "rust-workers");
6159        worker.register_replayed_workflow(
6160            "replay-counter",
6161            ReplayCounterState::default,
6162            |ctx, _input, state| async move {
6163                let loaded = ctx.activity("load-counter", json!([])).await?;
6164                state.update(|current| {
6165                    current.loaded = loaded.as_str().map(str::to_string);
6166                })?;
6167                for _ in 0..2 {
6168                    let signal = ctx.wait_signal("increment").await?;
6169                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
6170                    state.update(|current| current.count += amount)?;
6171                }
6172                state.update(|current| current.finished = true)?;
6173                state.read(|current| Ok(json!(current.count)))?
6174            },
6175        );
6176        worker.register_replayed_query::<ReplayCounterState, _, _>(
6177            "replay-counter",
6178            "current",
6179            |_ctx, state, _args| async move {
6180                Ok(json!({
6181                    "loaded": state.loaded,
6182                    "count": state.count,
6183                    "finished": state.finished,
6184                }))
6185            },
6186        );
6187        worker.register_replayed_query::<ReplayCounterState, _, _>(
6188            "replay-counter",
6189            "detached-mutation",
6190            |_ctx, state, _args| async move {
6191                let mut detached = (*state).clone();
6192                detached.count = 999;
6193                Ok(json!(detached.count))
6194            },
6195        );
6196        worker.register_replayed_query::<ReplayCounterState, _, _>(
6197            "replay-counter",
6198            "failed-mutation",
6199            |_ctx, state, _args| async move {
6200                let mut detached = (*state).clone();
6201                detached.count = 999;
6202                Err(Error::WorkerLoop("query refused".to_string()))
6203            },
6204        );
6205        worker
6206    }
6207
6208    fn replay_counter_query(
6209        query_name: &str,
6210        history_events: Value,
6211        run_status: &str,
6212    ) -> QueryTask {
6213        serde_json::from_value(json!({
6214            "query_task_id": format!("query-{query_name}"),
6215            "workflow_type": "replay-counter",
6216            "query_name": query_name,
6217            "payload_codec": "json",
6218            "workflow_arguments": {"codec": "json", "blob": "[]"},
6219            "query_arguments": {"codec": "json", "blob": "[]"},
6220            "history_events": history_events,
6221            "run_status": run_status,
6222        }))
6223        .expect("query task")
6224    }
6225
6226    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
6227        WorkflowContext {
6228            state: Arc::new(Mutex::new(
6229                WorkflowState::new_with_identity(
6230                    history,
6231                    None,
6232                    None,
6233                    "rust-workers".to_string(),
6234                    JSON_CODEC.to_string(),
6235                    None,
6236                )
6237                .expect("valid workflow history"),
6238            )),
6239        }
6240    }
6241
6242    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
6243        HistoryEvent {
6244            event_type: event_type.to_string(),
6245            payload,
6246            raw: HashMap::new(),
6247        }
6248    }
6249
6250    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
6251        vec![
6252            history_event(
6253                "ActivityScheduled",
6254                json!({
6255                    "sequence": 1,
6256                    "activity_type": "flaky",
6257                    "activity_execution_id": "act-1",
6258                    "activity": {
6259                        "id": "act-1",
6260                        "sequence": 1,
6261                        "type": "flaky",
6262                        "queue": "critical-activities",
6263                        "execution_mode": null,
6264                        "retry_policy": {
6265                            "snapshot_version": 1,
6266                            "max_attempts": 3,
6267                            "backoff_seconds": [2, 4],
6268                            "start_to_close_timeout": 30,
6269                            "schedule_to_start_timeout": 5,
6270                            "schedule_to_close_timeout": 90,
6271                            "heartbeat_timeout": 10,
6272                            "non_retryable_error_types": ["PermanentError"]
6273                        }
6274                    }
6275                }),
6276            ),
6277            history_event(
6278                "ActivityStarted",
6279                json!({
6280                    "sequence": 1,
6281                    "activity_type": "flaky",
6282                    "activity_execution_id": "act-1",
6283                    "activity_attempt_id": "attempt-1",
6284                    "attempt_number": 1
6285                }),
6286            ),
6287            history_event(
6288                "ActivityRetryScheduled",
6289                json!({
6290                    "sequence": 1,
6291                    "activity_type": "flaky",
6292                    "activity_execution_id": "act-1",
6293                    "activity_attempt_id": "attempt-1",
6294                    "attempt_number": 1,
6295                    "retry_after_attempt": 1,
6296                    "retry_backoff_seconds": 2,
6297                    "failure_category": "activity",
6298                    "exception_type": "TransientError"
6299                }),
6300            ),
6301            history_event(
6302                "ActivityStarted",
6303                json!({
6304                    "sequence": 1,
6305                    "activity_type": "flaky",
6306                    "activity_execution_id": "act-1",
6307                    "activity_attempt_id": "attempt-2",
6308                    "attempt_number": 2
6309                }),
6310            ),
6311            history_event(
6312                "ActivityCompleted",
6313                json!({
6314                    "sequence": 1,
6315                    "activity_type": "flaky",
6316                    "activity_execution_id": "act-1",
6317                    "activity_attempt_id": "attempt-2",
6318                    "attempt_number": 2,
6319                    "payload_codec": "json",
6320                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
6321                }),
6322            ),
6323        ]
6324    }
6325
6326    fn retry_activity_options() -> ActivityOptions {
6327        ActivityOptions::new()
6328            .task_queue("critical-activities")
6329            .retry_policy(
6330                ActivityRetryPolicy::new(3)
6331                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
6332                    .non_retryable_error_type("PermanentError"),
6333            )
6334            .start_to_close_timeout(Duration::from_secs(30))
6335            .schedule_to_start_timeout(Duration::from_secs(5))
6336            .schedule_to_close_timeout(Duration::from_secs(90))
6337            .heartbeat_timeout(Duration::from_secs(10))
6338    }
6339
6340    #[test]
6341    fn avro_generic_wrapper_round_trips_json_values() {
6342        let value = json!({"greeting": "hello", "count": 3, "ok": true});
6343        let envelope = PayloadEnvelope::avro(&value).expect("encode");
6344        assert_eq!(envelope.codec, DEFAULT_CODEC);
6345        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6346    }
6347
6348    #[test]
6349    fn json_codec_remains_plain_json() {
6350        let value = json!({"greeting": "hello", "count": 3, "ok": true});
6351        let envelope = PayloadEnvelope::json(&value).expect("encode");
6352
6353        assert_eq!(envelope.codec, JSON_CODEC);
6354        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
6355        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6356    }
6357
6358    #[test]
6359    fn typed_avro_payload_without_schema_context_keeps_diagnostic() {
6360        let envelope = PayloadEnvelope {
6361            codec: DEFAULT_CODEC.to_string(),
6362            blob: BASE64.encode([0x01]),
6363        };
6364
6365        let error = decode_payload::<Value>(&envelope).expect_err("typed payload must fail");
6366        assert_eq!(
6367            error.to_string(),
6368            "codec error: typed avro payloads require a schema context; v1 supports the generic wrapper"
6369        );
6370    }
6371
6372    #[test]
6373    fn workflow_context_schedules_activity_until_completion_is_in_history() {
6374        let ctx = WorkflowContext {
6375            state: Arc::new(Mutex::new(
6376                WorkflowState::new_with_identity(
6377                    Vec::new(),
6378                    Some("wf-parent".to_string()),
6379                    Some("run-parent".to_string()),
6380                    "rust-workers".to_string(),
6381                    DEFAULT_CODEC.to_string(),
6382                    None,
6383                )
6384                .expect("workflow state"),
6385            )),
6386        };
6387
6388        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
6389        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6390        assert!(matches!(
6391            call.as_mut().poll(&mut task_context),
6392            Poll::Pending
6393        ));
6394
6395        let commands = ctx.take_commands().expect("commands");
6396        assert_eq!(commands[0]["type"], "schedule_activity");
6397        assert_eq!(commands[0]["activity_type"], "hello.activity");
6398    }
6399
6400    #[test]
6401    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
6402        let ctx = workflow_context(Vec::new());
6403        let options = ActivityOptions::new()
6404            .task_queue("payments")
6405            .retry_policy(
6406                ActivityRetryPolicy::new(4)
6407                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
6408                    .non_retryable_error_type("ValidationError"),
6409            )
6410            .start_to_close_timeout(Duration::from_secs(120))
6411            .schedule_to_start_timeout(Duration::from_secs(10))
6412            .schedule_to_close_timeout(Duration::from_secs(300))
6413            .heartbeat_timeout(Duration::from_secs(15));
6414        let mut call = Box::pin(ctx.activity_with_options(
6415            "charge-card",
6416            options,
6417            json!([{"order_id": "o-1"}]),
6418        ));
6419        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6420
6421        assert!(matches!(
6422            call.as_mut().poll(&mut task_context),
6423            Poll::Pending
6424        ));
6425        assert!(matches!(
6426            call.as_mut().poll(&mut task_context),
6427            Poll::Pending
6428        ));
6429
6430        let commands = ctx.take_commands().expect("activity command");
6431        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
6432        assert_eq!(commands[0]["queue"], "payments");
6433        assert_eq!(
6434            commands[0]["retry_policy"],
6435            json!({
6436                "max_attempts": 4,
6437                "backoff_seconds": [1, 3, 9],
6438                "non_retryable_error_types": ["ValidationError"],
6439            })
6440        );
6441        assert_eq!(commands[0]["start_to_close_timeout"], 120);
6442        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
6443        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
6444        assert_eq!(commands[0]["heartbeat_timeout"], 15);
6445    }
6446
6447    #[test]
6448    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
6449        let ctx = workflow_context(Vec::new());
6450        let options = ActivityOptions::new().retry_policy(
6451            ActivityRetryPolicy::new(3)
6452                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
6453        );
6454        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6455        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6456
6457        assert!(matches!(
6458            call.as_mut().poll(&mut task_context),
6459            Poll::Pending
6460        ));
6461        assert_eq!(
6462            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
6463            json!([1, 2])
6464        );
6465    }
6466
6467    #[test]
6468    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
6469        let cases = [
6470            (
6471                ActivityOptions::new().task_queue("  "),
6472                ActivityOptionsErrorKind::EmptyTaskQueue,
6473            ),
6474            (
6475                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
6476                ActivityOptionsErrorKind::EmptyRetryPolicy,
6477            ),
6478            (
6479                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
6480                ActivityOptionsErrorKind::InvalidMaxAttempts,
6481            ),
6482            (
6483                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
6484                    max_attempts: None,
6485                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
6486                    non_retryable_error_types: Vec::new(),
6487                }),
6488                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
6489            ),
6490            (
6491                ActivityOptions::new().retry_policy(
6492                    ActivityRetryPolicy::new(2)
6493                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
6494                ),
6495                ActivityOptionsErrorKind::TooManyBackoffIntervals,
6496            ),
6497            (
6498                ActivityOptions::new().retry_policy(
6499                    ActivityRetryPolicy::new(2).exponential_backoff(
6500                        Duration::from_secs(1),
6501                        0,
6502                        None,
6503                    ),
6504                ),
6505                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
6506            ),
6507            (
6508                ActivityOptions::new()
6509                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
6510                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
6511            ),
6512            (
6513                ActivityOptions::new().retry_policy(
6514                    ActivityRetryPolicy::new(10_002).exponential_backoff(
6515                        Duration::from_secs(1),
6516                        1,
6517                        None,
6518                    ),
6519                ),
6520                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
6521            ),
6522            (
6523                ActivityOptions::new().retry_policy(
6524                    ActivityRetryPolicy::new(2)
6525                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
6526                ),
6527                ActivityOptionsErrorKind::BackoffOverflow,
6528            ),
6529        ];
6530
6531        for (options, expected_kind) in cases {
6532            let ctx = workflow_context(Vec::new());
6533            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6534            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6535            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
6536                call.as_mut().poll(&mut task_context)
6537            else {
6538                panic!("expected typed activity validation error");
6539            };
6540            assert_eq!(error.kind, expected_kind);
6541            assert!(ctx.take_commands().expect("commands").is_empty());
6542        }
6543    }
6544
6545    #[test]
6546    fn activity_options_validate_positive_and_ordered_timeouts() {
6547        let zero_timeout_cases = [
6548            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
6549            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
6550            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
6551            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
6552        ];
6553        for options in zero_timeout_cases {
6554            assert_eq!(
6555                options.validate().expect_err("zero timeout").kind,
6556                ActivityOptionsErrorKind::TimeoutNotPositive
6557            );
6558        }
6559
6560        let ordering_cases = [
6561            ActivityOptions::new()
6562                .heartbeat_timeout(Duration::from_secs(11))
6563                .start_to_close_timeout(Duration::from_secs(10)),
6564            ActivityOptions::new()
6565                .start_to_close_timeout(Duration::from_secs(31))
6566                .schedule_to_close_timeout(Duration::from_secs(30)),
6567            ActivityOptions::new()
6568                .schedule_to_start_timeout(Duration::from_secs(31))
6569                .schedule_to_close_timeout(Duration::from_secs(30)),
6570        ];
6571        for options in ordering_cases {
6572            assert_eq!(
6573                options.validate().expect_err("timeout order").kind,
6574                ActivityOptionsErrorKind::TimeoutOrder
6575            );
6576        }
6577
6578        assert_eq!(
6579            ActivityOptions::new()
6580                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
6581                .validate()
6582                .expect_err("protocol integer overflow")
6583                .kind,
6584            ActivityOptionsErrorKind::TimeoutOverflow
6585        );
6586    }
6587
6588    #[test]
6589    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
6590        let ctx = workflow_context(completed_retry_activity_history());
6591        let mut call =
6592            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6593        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6594
6595        assert!(matches!(
6596            call.as_mut().poll(&mut task_context),
6597            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6598        ));
6599        assert!(ctx.take_commands().expect("commands").is_empty());
6600        ctx.ensure_history_consumed().expect("history consumed");
6601    }
6602
6603    #[test]
6604    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
6605        let mut options = retry_activity_options();
6606        options
6607            .retry_policy
6608            .as_mut()
6609            .expect("retry policy")
6610            .non_retryable_error_types
6611            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
6612
6613        let new_ctx = workflow_context(Vec::new());
6614        let mut new_call =
6615            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
6616        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6617        assert!(matches!(
6618            new_call.as_mut().poll(&mut task_context),
6619            Poll::Pending
6620        ));
6621        let commands = new_ctx.take_commands().expect("commands");
6622        assert_eq!(commands.len(), 1);
6623        assert_eq!(
6624            commands[0]["retry_policy"]["non_retryable_error_types"],
6625            json!(["PermanentError"])
6626        );
6627
6628        let replay_ctx = workflow_context(completed_retry_activity_history());
6629        let mut replay_call =
6630            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
6631        assert!(matches!(
6632            replay_call.as_mut().poll(&mut task_context),
6633            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6634        ));
6635        assert!(replay_ctx.take_commands().expect("commands").is_empty());
6636        replay_ctx
6637            .ensure_history_consumed()
6638            .expect("history consumed");
6639    }
6640
6641    #[test]
6642    fn replayed_intermediate_retry_remains_pending_across_restarts() {
6643        let history = completed_retry_activity_history()
6644            .into_iter()
6645            .take(3)
6646            .collect::<Vec<_>>();
6647
6648        for _restart in 0..2 {
6649            let ctx = workflow_context(history.clone());
6650            let mut call =
6651                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6652            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6653            assert!(matches!(
6654                call.as_mut().poll(&mut task_context),
6655                Poll::Pending
6656            ));
6657            assert!(ctx.take_commands().expect("commands").is_empty());
6658        }
6659    }
6660
6661    #[test]
6662    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
6663        let mut changed_queue = retry_activity_options();
6664        changed_queue.task_queue = Some("different-queue".to_string());
6665
6666        let mut changed_max_attempts = retry_activity_options();
6667        let retry_policy = changed_max_attempts
6668            .retry_policy
6669            .as_mut()
6670            .expect("retry policy");
6671        retry_policy.max_attempts = Some(4);
6672
6673        let mut changed_backoff = retry_activity_options();
6674        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
6675        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
6676            Duration::from_secs(3),
6677            Duration::from_secs(4),
6678        ]));
6679
6680        let mut changed_non_retryable_types = retry_activity_options();
6681        let retry_policy = changed_non_retryable_types
6682            .retry_policy
6683            .as_mut()
6684            .expect("retry policy");
6685        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
6686
6687        let mut changed_start_to_close = retry_activity_options();
6688        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
6689        let mut changed_schedule_to_start = retry_activity_options();
6690        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
6691        let mut changed_schedule_to_close = retry_activity_options();
6692        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
6693        let mut changed_heartbeat = retry_activity_options();
6694        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
6695
6696        let cases = [
6697            (changed_queue, "activity_task_queue_mismatch"),
6698            (changed_max_attempts, "activity_retry_policy_mismatch"),
6699            (changed_backoff, "activity_retry_policy_mismatch"),
6700            (
6701                changed_non_retryable_types,
6702                "activity_retry_policy_mismatch",
6703            ),
6704            (changed_start_to_close, "activity_retry_policy_mismatch"),
6705            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
6706            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
6707            (changed_heartbeat, "activity_retry_policy_mismatch"),
6708        ];
6709
6710        for (options, expected_reason) in cases {
6711            let ctx = workflow_context(completed_retry_activity_history());
6712            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
6713            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6714            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6715                call.as_mut().poll(&mut task_context)
6716            else {
6717                panic!("changed activity options must fail replay");
6718            };
6719            assert_eq!(failure.reason, expected_reason);
6720            assert_eq!(failure.sequence, Some(1));
6721            assert!(ctx.take_commands().expect("commands").is_empty());
6722        }
6723    }
6724
6725    #[test]
6726    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
6727        let cases = [
6728            (
6729                "execution_mode",
6730                json!("local"),
6731                "activity_execution_mode_mismatch",
6732            ),
6733            (
6734                "snapshot_version",
6735                json!(2),
6736                "activity_retry_policy_mismatch",
6737            ),
6738        ];
6739
6740        for (field, value, expected_reason) in cases {
6741            let mut history = completed_retry_activity_history();
6742            let activity = history[0].payload["activity"]
6743                .as_object_mut()
6744                .expect("activity snapshot");
6745            if field == "execution_mode" {
6746                activity.insert(field.to_string(), value);
6747            } else {
6748                activity["retry_policy"]
6749                    .as_object_mut()
6750                    .expect("retry snapshot")
6751                    .insert(field.to_string(), value);
6752            }
6753
6754            let ctx = workflow_context(history);
6755            let mut call =
6756                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6757            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6758            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6759                call.as_mut().poll(&mut task_context)
6760            else {
6761                panic!("changed {field} must fail replay");
6762            };
6763            assert_eq!(failure.reason, expected_reason);
6764            assert_eq!(failure.sequence, Some(1));
6765            assert!(ctx.take_commands().expect("commands").is_empty());
6766        }
6767    }
6768
6769    #[test]
6770    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
6771        let mut history = completed_retry_activity_history();
6772        let activity = history[0].payload["activity"]
6773            .as_object_mut()
6774            .expect("activity snapshot");
6775        activity.remove("execution_mode");
6776        activity.remove("retry_policy");
6777
6778        let mut current = retry_activity_options();
6779        current.start_to_close_timeout = Some(Duration::from_secs(45));
6780        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
6781        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
6782        current.heartbeat_timeout = Some(Duration::from_secs(12));
6783
6784        let ctx = workflow_context(history);
6785        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
6786        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6787        assert!(matches!(
6788            call.as_mut().poll(&mut task_context),
6789            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6790        ));
6791        assert!(ctx.take_commands().expect("commands").is_empty());
6792        ctx.ensure_history_consumed().expect("history consumed");
6793    }
6794
6795    #[test]
6796    fn terminal_activity_failed_after_start_returns_typed_failure() {
6797        let history = vec![
6798            history_event(
6799                "ActivityScheduled",
6800                json!({
6801                    "sequence": 1,
6802                    "activity_type": "flaky",
6803                    "activity_execution_id": "act-terminal",
6804                    "activity": {
6805                        "id": "act-terminal",
6806                        "sequence": 1,
6807                        "type": "flaky",
6808                        "queue": "critical-activities",
6809                        "retry_policy": {
6810                            "snapshot_version": 1,
6811                            "max_attempts": 3,
6812                            "backoff_seconds": [2, 4],
6813                            "non_retryable_error_types": ["PermanentError"]
6814                        }
6815                    }
6816                }),
6817            ),
6818            history_event(
6819                "ActivityStarted",
6820                json!({
6821                    "sequence": 1,
6822                    "activity_type": "flaky",
6823                    "activity_execution_id": "act-terminal",
6824                    "activity_attempt_id": "attempt-1",
6825                    "attempt_number": 1
6826                }),
6827            ),
6828            history_event(
6829                "ActivityFailed",
6830                json!({
6831                    "sequence": 1,
6832                    "activity_type": "flaky",
6833                    "activity_execution_id": "act-terminal",
6834                    "activity_attempt_id": "attempt-1",
6835                    "attempt_number": 1,
6836                    "failure_id": "failure-terminal",
6837                    "failure_category": "activity",
6838                    "exception_type": "PermanentError",
6839                    "message": "cannot retry",
6840                    "non_retryable": true
6841                }),
6842            ),
6843        ];
6844        let ctx = workflow_context(history);
6845        let mut call =
6846            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6847        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6848
6849        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6850            call.as_mut().poll(&mut task_context)
6851        else {
6852            panic!("terminal ActivityFailed must settle the activity future");
6853        };
6854        assert_eq!(failure.kind, ActivityFailureKind::Failed);
6855        assert_eq!(
6856            failure.activity_execution_id.as_deref(),
6857            Some("act-terminal")
6858        );
6859        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
6860        assert!(failure.non_retryable);
6861        assert!(ctx.take_commands().expect("commands").is_empty());
6862        ctx.ensure_history_consumed().expect("history consumed");
6863    }
6864
6865    #[test]
6866    fn activity_terminal_events_return_machine_readable_failures() {
6867        let cases = [
6868            (
6869                "ActivityFailed",
6870                json!({
6871                    "sequence": 1,
6872                    "activity_type": "charge-card",
6873                    "activity_execution_id": "act-1",
6874                    "activity_attempt_id": "attempt-2",
6875                    "attempt_number": 2,
6876                    "failure_id": "failure-1",
6877                    "failure_category": "activity",
6878                    "exception_type": "PaymentDeclined",
6879                    "exception_class": "payments.PaymentDeclined",
6880                    "message": "card declined",
6881                    "non_retryable": true
6882                }),
6883                ActivityFailureKind::Failed,
6884                "activity",
6885            ),
6886            (
6887                "ActivityCancelled",
6888                json!({
6889                    "sequence": 1,
6890                    "activity_type": "charge-card",
6891                    "activity_execution_id": "act-1",
6892                    "activity_attempt_id": "attempt-1"
6893                }),
6894                ActivityFailureKind::Cancelled,
6895                "cancelled",
6896            ),
6897        ];
6898
6899        for (event_type, payload, expected_kind, expected_reason) in cases {
6900            let ctx = workflow_context(vec![history_event(event_type, payload)]);
6901            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
6902            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6903            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6904                call.as_mut().poll(&mut task_context)
6905            else {
6906                panic!("expected terminal activity failure");
6907            };
6908            assert_eq!(failure.kind, expected_kind);
6909            assert_eq!(failure.reason, expected_reason);
6910            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
6911            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
6912        }
6913    }
6914
6915    #[test]
6916    fn every_activity_timeout_class_is_typed() {
6917        for timeout_kind in [
6918            "start_to_close",
6919            "schedule_to_start",
6920            "schedule_to_close",
6921            "heartbeat",
6922        ] {
6923            let ctx = workflow_context(vec![history_event(
6924                "ActivityTimedOut",
6925                json!({
6926                    "sequence": 1,
6927                    "activity_type": "slow",
6928                    "activity_execution_id": "act-timeout",
6929                    "activity_attempt_id": "attempt-timeout",
6930                    "failure_category": "timeout",
6931                    "timeout_kind": timeout_kind,
6932                    "message": "deadline expired"
6933                }),
6934            )]);
6935            let mut call = Box::pin(ctx.activity("slow", json!([])));
6936            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6937            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6938                call.as_mut().poll(&mut task_context)
6939            else {
6940                panic!("expected timeout failure");
6941            };
6942            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
6943            assert_eq!(failure.reason, timeout_kind);
6944            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
6945            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
6946        }
6947    }
6948
6949    #[test]
6950    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
6951        let ctx = workflow_context(Vec::new());
6952        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
6953        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6954
6955        assert!(matches!(
6956            sleep.as_mut().poll(&mut task_context),
6957            Poll::Pending
6958        ));
6959        assert!(matches!(
6960            sleep.as_mut().poll(&mut task_context),
6961            Poll::Pending
6962        ));
6963
6964        let commands = ctx.take_commands().expect("timer command");
6965        assert_eq!(
6966            commands,
6967            vec![json!({
6968                "type": "start_timer",
6969                "delay_seconds": 2,
6970            })]
6971        );
6972    }
6973
6974    #[test]
6975    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
6976        let history = vec![
6977            history_event(
6978                "TimerScheduled",
6979                json!({
6980                    "sequence": 1,
6981                    "timer_id": "timer-1",
6982                    "delay_seconds": 5,
6983                    "fire_at": "2026-07-11T12:00:05Z",
6984                }),
6985            ),
6986            history_event(
6987                "TimerFired",
6988                json!({
6989                    "sequence": 1,
6990                    "timer_id": "timer-1",
6991                    "delay_seconds": 5,
6992                    "fire_at": "2026-07-11T12:00:05Z",
6993                    "fired_at": "2026-07-11T12:00:05Z",
6994                }),
6995            ),
6996        ];
6997
6998        for _restart in 0..2 {
6999            let ctx = workflow_context(history.clone());
7000            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
7001            let mut task_context = TaskContext::from_waker(noop_waker_ref());
7002            assert!(matches!(
7003                sleep.as_mut().poll(&mut task_context),
7004                Poll::Ready(Ok(()))
7005            ));
7006            assert!(ctx.take_commands().expect("commands").is_empty());
7007            ctx.ensure_history_consumed().expect("history consumed");
7008        }
7009    }
7010
7011    #[test]
7012    fn workflow_sleep_rejects_changed_delay_during_replay() {
7013        let ctx = workflow_context(vec![
7014            history_event(
7015                "TimerScheduled",
7016                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7017            ),
7018            history_event(
7019                "TimerFired",
7020                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7021            ),
7022        ]);
7023        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
7024        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7025
7026        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7027            sleep.as_mut().poll(&mut task_context)
7028        else {
7029            panic!("changed timer delay must be rejected");
7030        };
7031        assert_eq!(failure.reason, "timer_delay_mismatch");
7032        assert_eq!(failure.sequence, Some(1));
7033    }
7034
7035    #[test]
7036    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
7037        let lone_fire = WorkflowState::new(
7038            vec![history_event(
7039                "TimerFired",
7040                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7041            )],
7042            "rust-workers".to_string(),
7043            JSON_CODEC.to_string(),
7044            None,
7045        )
7046        .expect_err("TimerFired requires TimerScheduled");
7047        assert!(matches!(
7048            lone_fire,
7049            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7050                if reason == "timer_schedule_missing_or_duplicate"
7051        ));
7052
7053        let wrong_identity = WorkflowState::new(
7054            vec![
7055                history_event(
7056                    "TimerScheduled",
7057                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7058                ),
7059                history_event(
7060                    "TimerFired",
7061                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7062                ),
7063            ],
7064            "rust-workers".to_string(),
7065            JSON_CODEC.to_string(),
7066            None,
7067        )
7068        .expect_err("fire must match scheduled timer identity");
7069        assert!(matches!(
7070            wrong_identity,
7071            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7072                if reason == "timer_identity_mismatch"
7073        ));
7074
7075        let duplicate_fire = WorkflowState::new(
7076            vec![
7077                history_event(
7078                    "TimerScheduled",
7079                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7080                ),
7081                history_event(
7082                    "TimerFired",
7083                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7084                ),
7085                history_event(
7086                    "TimerFired",
7087                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7088                ),
7089            ],
7090            "rust-workers".to_string(),
7091            JSON_CODEC.to_string(),
7092            None,
7093        )
7094        .expect_err("a durable timer cannot fire twice");
7095        assert!(matches!(
7096            duplicate_fire,
7097            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7098                if reason == "duplicate_timer_fire"
7099        ));
7100
7101        let wrong_fired_delay = WorkflowState::new(
7102            vec![
7103                history_event(
7104                    "TimerScheduled",
7105                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7106                ),
7107                history_event(
7108                    "TimerFired",
7109                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
7110                ),
7111            ],
7112            "rust-workers".to_string(),
7113            JSON_CODEC.to_string(),
7114            None,
7115        )
7116        .expect_err("timer schedule and fire delays must agree");
7117        assert!(matches!(
7118            wrong_fired_delay,
7119            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7120                if reason == "timer_history_delay_mismatch"
7121        ));
7122    }
7123
7124    #[test]
7125    fn replay_rejects_activity_moved_before_recorded_timer() {
7126        let ctx = workflow_context(vec![
7127            history_event(
7128                "TimerScheduled",
7129                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7130            ),
7131            history_event(
7132                "TimerFired",
7133                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7134            ),
7135            history_event(
7136                "ActivityCompleted",
7137                json!({
7138                    "sequence": 2,
7139                    "activity_type": "after-timer",
7140                    "payload_codec": "json",
7141                    "result": {"codec": "json", "blob": "\"done\""},
7142                }),
7143            ),
7144        ]);
7145        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
7146        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7147
7148        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7149            activity.as_mut().poll(&mut task_context)
7150        else {
7151            panic!("reordered durable command must be rejected");
7152        };
7153        assert_eq!(failure.reason, "recorded_command_mismatch");
7154        assert_eq!(failure.sequence, Some(1));
7155        assert_eq!(failure.expected.as_deref(), Some("timer"));
7156        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
7157    }
7158
7159    #[test]
7160    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
7161        let signal_then_timer = vec![
7162            history_event(
7163                "ConditionWaitOpened",
7164                json!({"sequence": 1, "condition_key": "signal:go"}),
7165            ),
7166            history_event(
7167                "SignalReceived",
7168                json!({
7169                    "signal_name": "go",
7170                    "arguments": ["now"],
7171                }),
7172            ),
7173            history_event(
7174                "TimerScheduled",
7175                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7176            ),
7177            history_event(
7178                "TimerFired",
7179                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7180            ),
7181        ];
7182
7183        let ctx = workflow_context(signal_then_timer.clone());
7184        let mut signal = Box::pin(ctx.wait_signal("go"));
7185        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7186        assert!(matches!(
7187            signal.as_mut().poll(&mut task_context),
7188            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
7189        ));
7190        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
7191        assert!(matches!(
7192            timer.as_mut().poll(&mut task_context),
7193            Poll::Ready(Ok(()))
7194        ));
7195        ctx.ensure_history_consumed()
7196            .expect("signal and timer history consumed in order");
7197
7198        let reordered = workflow_context(signal_then_timer);
7199        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
7200        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7201            timer_first.as_mut().poll(&mut task_context)
7202        else {
7203            panic!("timer cannot consume signal-wait-first history");
7204        };
7205        assert_eq!(failure.reason, "recorded_command_mismatch");
7206        assert_eq!(failure.sequence, Some(1));
7207        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
7208
7209        let timer_then_signal = vec![
7210            history_event(
7211                "TimerScheduled",
7212                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7213            ),
7214            history_event(
7215                "TimerFired",
7216                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7217            ),
7218            history_event(
7219                "ConditionWaitOpened",
7220                json!({"sequence": 2, "condition_key": "signal:go"}),
7221            ),
7222            history_event(
7223                "SignalReceived",
7224                json!({"signal_name": "go", "arguments": []}),
7225            ),
7226        ];
7227        let reordered = workflow_context(timer_then_signal);
7228        let mut signal_first = Box::pin(reordered.wait_signal("go"));
7229        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7230            signal_first.as_mut().poll(&mut task_context)
7231        else {
7232            panic!("signal wait cannot consume timer-first history");
7233        };
7234        assert_eq!(failure.reason, "recorded_command_mismatch");
7235        assert_eq!(failure.sequence, Some(1));
7236        assert_eq!(failure.expected.as_deref(), Some("timer"));
7237    }
7238
7239    #[test]
7240    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
7241        let duplicate_timer = WorkflowState::new(
7242            vec![
7243                history_event(
7244                    "TimerScheduled",
7245                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7246                ),
7247                history_event(
7248                    "TimerScheduled",
7249                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7250                ),
7251            ],
7252            "rust-workers".to_string(),
7253            JSON_CODEC.to_string(),
7254            None,
7255        )
7256        .expect_err("one workflow sequence cannot schedule two timers");
7257        assert!(matches!(
7258            duplicate_timer,
7259            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7260                if reason == "timer_schedule_missing_or_duplicate"
7261        ));
7262
7263        let colliding_kinds = WorkflowState::new(
7264            vec![
7265                history_event(
7266                    "TimerScheduled",
7267                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7268                ),
7269                history_event(
7270                    "ActivityCompleted",
7271                    json!({"sequence": 1, "activity_type": "same-sequence"}),
7272                ),
7273            ],
7274            "rust-workers".to_string(),
7275            JSON_CODEC.to_string(),
7276            None,
7277        )
7278        .expect_err("one workflow sequence cannot identify two command kinds");
7279        assert!(matches!(
7280            colliding_kinds,
7281            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7282                if reason == "durable_command_sequence_collision"
7283        ));
7284
7285        let duplicate_signal_wait = WorkflowState::new(
7286            vec![
7287                history_event(
7288                    "SignalWaitOpened",
7289                    json!({"sequence": 1, "signal_name": "go"}),
7290                ),
7291                history_event(
7292                    "SignalWaitOpened",
7293                    json!({"sequence": 1, "signal_name": "go"}),
7294                ),
7295            ],
7296            "rust-workers".to_string(),
7297            JSON_CODEC.to_string(),
7298            None,
7299        )
7300        .expect_err("one workflow sequence cannot open two signal waits");
7301        assert!(matches!(
7302            duplicate_signal_wait,
7303            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7304                if reason == "duplicate_signal_wait_open"
7305        ));
7306    }
7307
7308    #[test]
7309    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
7310        let ctx = workflow_context(Vec::new());
7311        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
7312        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7313        assert!(matches!(
7314            sleep.as_mut().poll(&mut task_context),
7315            Poll::Ready(Err(Error::TimerDurationOverflow))
7316        ));
7317        assert!(ctx.take_commands().expect("commands").is_empty());
7318    }
7319
7320    #[test]
7321    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
7322        let client = Client::new("http://127.0.0.1:8080").expect("client");
7323        let mut worker = Worker::new(client, "rust-workers");
7324        worker.register_workflow("rust.timer", |ctx, _input| async move {
7325            ctx.sleep(Duration::from_secs(5)).await?;
7326            ctx.activity("after-timer", json!([])).await
7327        });
7328
7329        let task = |history_events| WorkflowTask {
7330            task_id: "wft-rust-timer-1".to_string(),
7331            workflow_id: Some("wf-rust-timer".to_string()),
7332            run_id: Some("run-rust-timer".to_string()),
7333            workflow_type: "rust.timer".to_string(),
7334            payload_codec: JSON_CODEC.to_string(),
7335            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7336            history_events,
7337            total_history_events: None,
7338            next_history_page_token: None,
7339            workflow_task_attempt: 1,
7340            workflow_signal_id: None,
7341            signal_name: None,
7342            signal_arguments: None,
7343            lease_owner: Some("rust-worker".to_string()),
7344        };
7345
7346        let initial = worker
7347            .execute_workflow_task(task(Vec::new()))
7348            .expect("initial timer task");
7349        assert_eq!(
7350            initial,
7351            vec![json!({"type": "start_timer", "delay_seconds": 5})]
7352        );
7353
7354        let replayed = worker
7355            .execute_workflow_task(task(vec![
7356                history_event(
7357                    "TimerScheduled",
7358                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7359                ),
7360                history_event(
7361                    "TimerFired",
7362                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7363                ),
7364                history_event(
7365                    "ActivityCompleted",
7366                    json!({
7367                        "sequence": 2,
7368                        "activity_type": "after-timer",
7369                        "payload_codec": "json",
7370                        "result": {"codec": "json", "blob": "\"done\""},
7371                    }),
7372                ),
7373            ]))
7374            .expect("replayed workflow task");
7375        assert_eq!(replayed.len(), 1);
7376        assert_eq!(replayed[0]["type"], "complete_workflow");
7377        assert_eq!(
7378            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
7379            json!("done")
7380        );
7381    }
7382
7383    #[test]
7384    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
7385        let client = Client::new("http://127.0.0.1:8080").expect("client");
7386        let mut worker = Worker::new(client, "rust-workers");
7387        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
7388            ctx.sleep(Duration::from_secs(5)).await?;
7389            Ok(json!({"status": "timer fired"}))
7390        });
7391
7392        let task = WorkflowTask {
7393            task_id: "wft-rust-timer-pending".to_string(),
7394            workflow_id: Some("wf-rust-timer".to_string()),
7395            run_id: Some("run-rust-timer".to_string()),
7396            workflow_type: "rust.timer.pending".to_string(),
7397            payload_codec: JSON_CODEC.to_string(),
7398            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7399            history_events: vec![history_event(
7400                "TimerScheduled",
7401                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7402            )],
7403            total_history_events: Some(1),
7404            next_history_page_token: None,
7405            workflow_task_attempt: 1,
7406            workflow_signal_id: None,
7407            signal_name: None,
7408            signal_arguments: None,
7409            lease_owner: Some("rust-worker".to_string()),
7410        };
7411
7412        for _redelivery_or_restart in 0..2 {
7413            let commands = worker
7414                .execute_workflow_task(task.clone())
7415                .expect("recorded timer remains pending");
7416            assert!(
7417                commands.is_empty(),
7418                "recorded timer must not be rescheduled"
7419            );
7420        }
7421    }
7422
7423    #[test]
7424    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
7425        let client = Client::new("http://127.0.0.1:8080").expect("client");
7426        let mut worker = Worker::new(client, "rust-workers");
7427        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
7428            Ok(json!({"status": "completed"}))
7429        });
7430        let task = WorkflowTask {
7431            task_id: "wft-rust-timer-removed".to_string(),
7432            workflow_id: Some("wf-rust-timer".to_string()),
7433            run_id: Some("run-rust-timer".to_string()),
7434            workflow_type: "rust.timer.removed".to_string(),
7435            payload_codec: JSON_CODEC.to_string(),
7436            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7437            history_events: vec![
7438                history_event(
7439                    "TimerScheduled",
7440                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7441                ),
7442                history_event(
7443                    "TimerFired",
7444                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7445                ),
7446            ],
7447            total_history_events: Some(2),
7448            next_history_page_token: None,
7449            workflow_task_attempt: 1,
7450            workflow_signal_id: None,
7451            signal_name: None,
7452            signal_arguments: None,
7453            lease_owner: Some("rust-worker".to_string()),
7454        };
7455
7456        let Error::NonDeterministicReplay(failure) = worker
7457            .execute_workflow_task(task)
7458            .expect_err("removed timer must fail replay")
7459        else {
7460            panic!("expected typed replay failure");
7461        };
7462        assert_eq!(failure.reason, "recorded_commands_unconsumed");
7463        assert_eq!(failure.sequence, Some(1));
7464    }
7465
7466    #[test]
7467    fn workflow_context_emits_explicit_child_workflow_contract() {
7468        let ctx = WorkflowContext {
7469            state: Arc::new(Mutex::new(
7470                WorkflowState::new_with_identity(
7471                    Vec::new(),
7472                    Some("wf-parent".to_string()),
7473                    Some("run-parent".to_string()),
7474                    "parent-workers".to_string(),
7475                    JSON_CODEC.to_string(),
7476                    None,
7477                )
7478                .expect("workflow state"),
7479            )),
7480        };
7481        let options = ChildWorkflowOptions::new("python-workers")
7482            .parent_close_policy(ParentClosePolicy::RequestCancel)
7483            .retry_policy(ChildWorkflowRetryPolicy {
7484                max_attempts: Some(3),
7485                backoff_seconds: vec![1, 5],
7486                non_retryable_error_types: vec!["ValidationError".to_string()],
7487            })
7488            .execution_timeout_seconds(600)
7489            .run_timeout_seconds(120);
7490        let mut call = Box::pin(ctx.start_child_workflow(
7491            "python.fulfil-order",
7492            options,
7493            json!([{"order_id": "order-42"}]),
7494        ));
7495        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7496
7497        assert!(matches!(
7498            call.as_mut().poll(&mut task_context),
7499            Poll::Pending
7500        ));
7501        let commands = ctx.take_commands().expect("commands");
7502        assert_eq!(commands.len(), 1);
7503        let command = &commands[0];
7504        assert_eq!(command["type"], "start_child_workflow");
7505        assert_eq!(command["workflow_type"], "python.fulfil-order");
7506        assert_eq!(command["queue"], "python-workers");
7507        assert_eq!(command["parent_close_policy"], "request_cancel");
7508        assert_eq!(command["retry_policy"]["max_attempts"], 3);
7509        assert_eq!(command["execution_timeout_seconds"], 600);
7510        assert_eq!(command["run_timeout_seconds"], 120);
7511        assert_eq!(
7512            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
7513            json!([{"order_id": "order-42"}])
7514        );
7515    }
7516
7517    fn child_parent_worker() -> Worker {
7518        let client = Client::new("http://127.0.0.1:8080").expect("client");
7519        let mut worker = Worker::new(client, "rust-parent-workers");
7520        worker.register_workflow("rust.parent", |ctx, _input| async move {
7521            let child = ctx
7522                .start_child_workflow(
7523                    "python.child",
7524                    ChildWorkflowOptions::new("python-child-workers")
7525                        .parent_close_policy(ParentClosePolicy::Terminate),
7526                    json!([{"codec_probe": [1, true, "rust"]}]),
7527                )
7528                .await?;
7529            Ok(json!({
7530                "parent_workflow_id": child.parent.workflow_id,
7531                "parent_run_id": child.parent.run_id,
7532                "child_workflow_id": child.child.workflow_id,
7533                "child_run_id": child.child.run_id,
7534                "child_workflow_type": child.child_workflow_type,
7535                "result": child.result,
7536            }))
7537        });
7538        worker
7539    }
7540
7541    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
7542        WorkflowTask {
7543            task_id: "wft-child-parent".to_string(),
7544            workflow_id: Some("wf-parent".to_string()),
7545            run_id: Some("run-parent".to_string()),
7546            workflow_type: "rust.parent".to_string(),
7547            payload_codec: JSON_CODEC.to_string(),
7548            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
7549            history_events: vec![
7550                HistoryEvent {
7551                    event_type: "ChildWorkflowScheduled".to_string(),
7552                    payload: json!({
7553                        "sequence": 1,
7554                        "child_call_id": "call-child",
7555                        "child_workflow_instance_id": "wf-child",
7556                        "child_workflow_run_id": "run-child",
7557                        "child_workflow_type": "python.child",
7558                    }),
7559                    raw: HashMap::new(),
7560                },
7561                HistoryEvent {
7562                    event_type: event_type.to_string(),
7563                    payload,
7564                    raw: HashMap::new(),
7565                },
7566            ],
7567            total_history_events: Some(2),
7568            next_history_page_token: None,
7569            workflow_task_attempt: 1,
7570            workflow_signal_id: None,
7571            signal_name: None,
7572            signal_arguments: None,
7573            lease_owner: Some("rust-worker".to_string()),
7574        }
7575    }
7576
7577    #[test]
7578    fn committed_child_result_replays_without_starting_a_duplicate() {
7579        let worker = child_parent_worker();
7580        let task = child_parent_task(
7581            "ChildRunCompleted",
7582            json!({
7583                "sequence": 1,
7584                "child_call_id": "call-child",
7585                "child_workflow_instance_id": "wf-child",
7586                "child_workflow_run_id": "run-child",
7587                "child_workflow_type": "python.child",
7588                "payload_codec": "json",
7589                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
7590            }),
7591        );
7592
7593        for _restart in 0..2 {
7594            let commands = worker
7595                .execute_workflow_task(task.clone())
7596                .expect("replayed parent task");
7597            assert_eq!(commands.len(), 1);
7598            assert_eq!(commands[0]["type"], "complete_workflow");
7599            assert!(!commands
7600                .iter()
7601                .any(|command| command["type"] == "start_child_workflow"));
7602            let output =
7603                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7604            assert_eq!(output["parent_workflow_id"], "wf-parent");
7605            assert_eq!(output["parent_run_id"], "run-parent");
7606            assert_eq!(output["child_workflow_id"], "wf-child");
7607            assert_eq!(output["child_run_id"], "run-child");
7608            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
7609        }
7610    }
7611
7612    #[test]
7613    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
7614        let worker = child_parent_worker();
7615        let mut task = child_parent_task("unused", Value::Null);
7616        task.history_events.truncate(1);
7617        task.total_history_events = Some(1);
7618
7619        for _redelivery_or_restart in 0..2 {
7620            let commands = worker
7621                .execute_workflow_task(task.clone())
7622                .expect("recorded child remains pending");
7623            assert!(
7624                commands.is_empty(),
7625                "recorded pending child must not be started again"
7626            );
7627        }
7628    }
7629
7630    #[test]
7631    fn child_cancellation_becomes_stable_parent_failure_command() {
7632        let worker = child_parent_worker();
7633        let task = child_parent_task(
7634            "ChildRunCancelled",
7635            json!({
7636                "sequence": 1,
7637                "child_workflow_instance_id": "wf-child",
7638                "child_workflow_run_id": "run-child",
7639                "child_workflow_type": "python.child",
7640                "failure_id": "failure-child",
7641                "failure_category": "cancelled",
7642                "message": "cancelled by parent-close policy",
7643            }),
7644        );
7645
7646        let commands = worker
7647            .execute_workflow_task(task)
7648            .expect("parent settlement");
7649        assert_eq!(commands.len(), 1);
7650        assert_eq!(commands[0]["type"], "fail_workflow");
7651        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
7652        assert_eq!(
7653            commands[0]["exception"]["properties"]["reason"],
7654            "cancelled"
7655        );
7656        assert_eq!(
7657            commands[0]["exception"]["properties"]["child_workflow_run_id"],
7658            "run-child"
7659        );
7660    }
7661
7662    #[test]
7663    fn workflow_can_handle_typed_child_failure() {
7664        let client = Client::new("http://127.0.0.1:8080").expect("client");
7665        let mut worker = Worker::new(client, "rust-parent-workers");
7666        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
7667            match ctx
7668                .start_child_workflow(
7669                    "python.child",
7670                    ChildWorkflowOptions::new("python-child-workers"),
7671                    json!([]),
7672                )
7673                .await
7674            {
7675                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
7676                    "reason": failure.reason,
7677                    "failure_id": failure.failure_id,
7678                    "exception_class": failure.exception_class,
7679                    "child_run_id": failure.child_workflow_run_id,
7680                })),
7681                Err(error) => Err(error),
7682                Ok(_) => Err(Error::WorkerLoop(
7683                    "child unexpectedly succeeded".to_string(),
7684                )),
7685            }
7686        });
7687        let mut task = child_parent_task(
7688            "ChildRunFailed",
7689            json!({
7690                "sequence": 1,
7691                "child_workflow_instance_id": "wf-child",
7692                "child_workflow_run_id": "run-child",
7693                "child_workflow_type": "python.child",
7694                "failure_id": "failure-child",
7695                "failure_category": "child_workflow",
7696                "message": "payment rejected",
7697                "exception": {
7698                    "type": "PaymentRejected",
7699                    "class": "payments.PaymentRejected",
7700                    "message": "payment rejected"
7701                }
7702            }),
7703        );
7704        task.workflow_type = "rust.handled-parent".to_string();
7705
7706        let commands = worker.execute_workflow_task(task).expect("handled failure");
7707        assert_eq!(commands[0]["type"], "complete_workflow");
7708        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7709        assert_eq!(output["reason"], "child_workflow");
7710        assert_eq!(output["failure_id"], "failure-child");
7711        assert_eq!(output["exception_class"], "payments.PaymentRejected");
7712        assert_eq!(output["child_run_id"], "run-child");
7713    }
7714
7715    #[test]
7716    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
7717        let client = Client::new("http://127.0.0.1:8080").expect("client");
7718        let mut worker = Worker::new(client, "rust-workers");
7719
7720        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
7721            let signal = ctx.wait_signal("start").await?;
7722            let name = signal
7723                .first()
7724                .and_then(|value| value.as_str())
7725                .unwrap_or("world");
7726            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
7727            Ok(json!({
7728                "greeting": greeting,
7729                "language": "rust"
7730            }))
7731        });
7732
7733        let signal_arguments =
7734            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
7735        let task = WorkflowTask {
7736            task_id: "wft-rust-signal-1".to_string(),
7737            workflow_id: Some("wf-rust-hello".to_string()),
7738            run_id: Some("run-rust-hello".to_string()),
7739            workflow_type: "rust.hello_workflow".to_string(),
7740            payload_codec: DEFAULT_CODEC.to_string(),
7741            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7742            history_events: vec![HistoryEvent {
7743                event_type: "SignalReceived".to_string(),
7744                payload: json!({
7745                    "signal_id": "sig-rust-1",
7746                    "signal_name": "start"
7747                }),
7748                raw: HashMap::new(),
7749            }],
7750            total_history_events: Some(1),
7751            next_history_page_token: None,
7752            workflow_task_attempt: 1,
7753            workflow_signal_id: Some("sig-rust-1".to_string()),
7754            signal_name: Some("start".to_string()),
7755            signal_arguments: Some(signal_arguments),
7756            lease_owner: Some("rust-worker".to_string()),
7757        };
7758
7759        let commands = worker.execute_workflow_task(task).expect("workflow task");
7760
7761        assert_eq!(commands.len(), 1);
7762        assert_eq!(commands[0]["type"], "schedule_activity");
7763        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
7764        assert_eq!(
7765            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
7766            json!(["Rust"])
7767        );
7768    }
7769
7770    #[test]
7771    fn workflow_task_appends_paginated_history_events() {
7772        let mut task = WorkflowTask {
7773            task_id: "wft-rust-pages-1".to_string(),
7774            workflow_id: Some("wf-rust-pages".to_string()),
7775            run_id: Some("run-rust-pages".to_string()),
7776            workflow_type: "rust.hello_workflow".to_string(),
7777            payload_codec: DEFAULT_CODEC.to_string(),
7778            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7779            history_events: vec![HistoryEvent {
7780                event_type: "WorkflowStarted".to_string(),
7781                payload: json!({}),
7782                raw: HashMap::new(),
7783            }],
7784            total_history_events: Some(3),
7785            next_history_page_token: Some("MQ==".to_string()),
7786            workflow_task_attempt: 1,
7787            workflow_signal_id: None,
7788            signal_name: None,
7789            signal_arguments: None,
7790            lease_owner: Some("rust-worker".to_string()),
7791        };
7792
7793        task.append_history_page(WorkflowTaskHistoryPage {
7794            history_events: vec![
7795                HistoryEvent {
7796                    event_type: "SignalReceived".to_string(),
7797                    payload: json!({
7798                        "signal_id": "sig-rust-1",
7799                        "signal_name": "start",
7800                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
7801                            .expect("signal arguments")
7802                    }),
7803                    raw: HashMap::new(),
7804                },
7805                HistoryEvent {
7806                    event_type: "MarkerRecorded".to_string(),
7807                    payload: json!({"sequence": 3}),
7808                    raw: HashMap::new(),
7809                },
7810            ],
7811            total_history_events: Some(3),
7812            next_history_page_token: None,
7813        });
7814
7815        assert_eq!(task.history_events.len(), 3);
7816        assert_eq!(task.total_history_events, Some(3));
7817        assert_eq!(task.next_history_page_token, None);
7818
7819        let signals =
7820            signal_values(&task.history_events, "start", DEFAULT_CODEC, None).expect("signals");
7821        assert_eq!(signals, vec![vec![json!("Rust")]]);
7822    }
7823
7824    #[tokio::test]
7825    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
7826        let client = Client::new("http://127.0.0.1:8080").expect("client");
7827        let mut worker = Worker::new(client, "rust-workers");
7828        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
7829        worker.register_query("counter", "current", |ctx, _args| async move {
7830            let mut count = 0_i64;
7831            for signal in ctx.signal_events() {
7832                let value = signal
7833                    .arguments
7834                    .first()
7835                    .and_then(Value::as_i64)
7836                    .unwrap_or_default();
7837                match signal.name.as_str() {
7838                    "increment" => count += value,
7839                    "set" => count = value,
7840                    _ => {}
7841                }
7842            }
7843            Ok(json!(count))
7844        });
7845
7846        let task = QueryTask {
7847            query_task_id: "query-rust-counter".to_string(),
7848            query_task_attempt: 1,
7849            lease_owner: Some("rust-worker".to_string()),
7850            workflow_id: Some("counter-1".to_string()),
7851            run_id: Some("run-counter-1".to_string()),
7852            workflow_type: "counter".to_string(),
7853            query_name: "current".to_string(),
7854            payload_codec: DEFAULT_CODEC.to_string(),
7855            workflow_arguments: Some(
7856                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
7857            ),
7858            query_arguments: Some(
7859                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
7860            ),
7861            history_events: vec![
7862                HistoryEvent {
7863                    event_type: "SignalReceived".to_string(),
7864                    payload: json!({
7865                        "signal_id": "php-signal-1",
7866                        "signal_name": "increment",
7867                        "workflow_sequence": 1,
7868                        "payload_codec": DEFAULT_CODEC,
7869                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
7870                    }),
7871                    raw: HashMap::new(),
7872                },
7873                HistoryEvent {
7874                    event_type: "SignalReceived".to_string(),
7875                    payload: json!({
7876                        "signal_id": "python-signal-2",
7877                        "signal_name": "increment",
7878                        "workflow_sequence": 2,
7879                        "payload_codec": JSON_CODEC,
7880                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
7881                    }),
7882                    raw: HashMap::new(),
7883                },
7884                HistoryEvent {
7885                    event_type: "SignalReceived".to_string(),
7886                    payload: json!({
7887                        "signal_id": "rust-signal-3",
7888                        "signal_name": "set",
7889                        "workflow_sequence": 3,
7890                        "payload_codec": DEFAULT_CODEC,
7891                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
7892                    }),
7893                    raw: HashMap::new(),
7894                },
7895            ],
7896            history_export: None,
7897            run_status: Some("completed".to_string()),
7898        };
7899
7900        let result = worker.execute_query_task(task).await.expect("query result");
7901        assert_eq!(result, json!(0));
7902    }
7903
7904    #[tokio::test]
7905    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
7906        let worker = replay_counter_worker();
7907        let running_history = json!([
7908            {
7909                "type": "ActivityCompleted",
7910                "payload": {
7911                    "sequence": 1,
7912                    "activity_type": "load-counter",
7913                    "payload_codec": "json",
7914                    "result": {"codec": "json", "blob": "\"loaded\""}
7915                }
7916            },
7917            {
7918                "type": "SignalReceived",
7919                "payload": {
7920                    "signal_id": "signal-3",
7921                    "signal_name": "increment",
7922                    "payload_codec": "json",
7923                    "arguments": {"codec": "json", "blob": "[3]"}
7924                }
7925            }
7926        ]);
7927
7928        let running = worker
7929            .execute_query_task(replay_counter_query(
7930                "current",
7931                running_history.clone(),
7932                "running",
7933            ))
7934            .await
7935            .expect("running replay query");
7936        assert_eq!(
7937            running,
7938            json!({"loaded": "loaded", "count": 3, "finished": false})
7939        );
7940
7941        let detached = worker
7942            .execute_query_task(replay_counter_query(
7943                "detached-mutation",
7944                running_history.clone(),
7945                "running",
7946            ))
7947            .await
7948            .expect("query mutates only its detached state clone");
7949        assert_eq!(detached, json!(999));
7950        let failed = worker
7951            .execute_query_task(replay_counter_query(
7952                "failed-mutation",
7953                running_history.clone(),
7954                "running",
7955            ))
7956            .await
7957            .expect_err("failed query");
7958        assert_eq!(failed.reason, "query_rejected");
7959        let unchanged = worker
7960            .execute_query_task(replay_counter_query("current", running_history, "running"))
7961            .await
7962            .expect("later query reconstructs unchanged state");
7963        assert_eq!(unchanged, running);
7964
7965        let restarted_worker = replay_counter_worker();
7966        let restarted_task: QueryTask = serde_json::from_value(json!({
7967            "query_task_id": "query-after-restart",
7968            "workflow_id": "counter-1",
7969            "run_id": "run-counter-1",
7970            "workflow_type": "replay-counter",
7971            "query_name": "current",
7972            "payload_codec": "json",
7973            "workflow_arguments": {"codec": "json", "blob": "[]"},
7974            "query_arguments": {"codec": "json", "blob": "[]"},
7975            "history_events": [],
7976            "history_export": {
7977                "payloads": {"codec": "json"},
7978                "history_events": [
7979                    {
7980                        "type": "ActivityCompleted",
7981                        "payload": {
7982                            "sequence": 1,
7983                            "activity_type": "load-counter",
7984                            "payload_codec": "json",
7985                            "result": null
7986                        }
7987                    },
7988                    {
7989                        "type": "SignalReceived",
7990                        "payload": {"signal_id": "signal-3", "signal_name": "increment"}
7991                    },
7992                    {
7993                        "type": "SignalReceived",
7994                        "payload": {"signal_id": "signal-5", "signal_name": "increment"}
7995                    }
7996                ],
7997                "activities": [{
7998                    "sequence": 1,
7999                    "activity_type": "load-counter",
8000                    "payload_codec": "json",
8001                    "result": {"codec": "json", "blob": "\"loaded\""}
8002                }],
8003                "signals": [
8004                    {
8005                        "id": "signal-3",
8006                        "name": "increment",
8007                        "payload_codec": "json",
8008                        "arguments": "[3]"
8009                    },
8010                    {
8011                        "id": "signal-5",
8012                        "name": "increment",
8013                        "payload_codec": "json",
8014                        "arguments": "[5]"
8015                    }
8016                ]
8017            },
8018            "run_status": "completed"
8019        }))
8020        .expect("cold replay query task");
8021        let completed = restarted_worker
8022            .execute_query_task(restarted_task)
8023            .await
8024            .expect("completed cold replay query");
8025        assert_eq!(
8026            completed,
8027            json!({"loaded": "loaded", "count": 8, "finished": true})
8028        );
8029    }
8030
8031    #[tokio::test]
8032    async fn replayed_query_replay_failures_are_machine_readable() {
8033        let worker = replay_counter_worker();
8034        let task = replay_counter_query(
8035            "current",
8036            json!([{
8037                "type": "ActivityCompleted",
8038                "payload": {
8039                    "sequence": 1,
8040                    "payload_codec": "json",
8041                    "result": {"codec": "json", "blob": "{"}
8042                }
8043            }]),
8044            "running",
8045        );
8046        let failure = worker
8047            .execute_query_task(task)
8048            .await
8049            .expect_err("invalid replay history payload");
8050        assert_eq!(failure.reason, "query_workflow_state_unavailable");
8051        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
8052    }
8053
8054    #[tokio::test]
8055    async fn query_task_restores_compact_history_from_export() {
8056        let client = Client::new("http://127.0.0.1:8080").expect("client");
8057        let mut worker = Worker::new(client, "rust-workers");
8058        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8059        worker.register_query("counter", "current", |ctx, _args| async move {
8060            Ok(json!(ctx.signals("increment")[0][0]))
8061        });
8062        let task: QueryTask = serde_json::from_value(json!({
8063            "query_task_id": "query-export",
8064            "workflow_type": "counter",
8065            "query_name": "current",
8066            "payload_codec": "json",
8067            "workflow_arguments": {"codec": "json", "blob": "[]"},
8068            "query_arguments": {"codec": "json", "blob": "[]"},
8069            "history_events": [],
8070            "history_export": {
8071                "payloads": {"codec": "json"},
8072                "history_events": [{
8073                    "type": "SignalReceived",
8074                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
8075                }],
8076                "signals": [{
8077                    "id": "signal-export",
8078                    "name": "increment",
8079                    "status": "applied",
8080                    "workflow_sequence": 1,
8081                    "payload_codec": "json",
8082                    "arguments": "[9]"
8083                }]
8084            }
8085        }))
8086        .expect("query task");
8087
8088        let result = worker.execute_query_task(task).await.expect("query result");
8089        assert_eq!(result, json!(9));
8090    }
8091
8092    #[tokio::test]
8093    async fn query_task_failures_have_stable_reasons() {
8094        let client = Client::new("http://127.0.0.1:8080").expect("client");
8095        let mut worker = Worker::new(client, "rust-workers");
8096        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8097        worker.register_query(
8098            "counter",
8099            "current",
8100            |_ctx, _args| async move { Ok(json!(0)) },
8101        );
8102
8103        let base_task = QueryTask {
8104            query_task_id: "query-errors".to_string(),
8105            query_task_attempt: 1,
8106            lease_owner: None,
8107            workflow_id: Some("counter-errors".to_string()),
8108            run_id: Some("run-errors".to_string()),
8109            workflow_type: "counter".to_string(),
8110            query_name: "missing".to_string(),
8111            payload_codec: JSON_CODEC.to_string(),
8112            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8113            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8114            history_events: Vec::new(),
8115            history_export: None,
8116            run_status: Some("running".to_string()),
8117        };
8118
8119        let unknown = worker
8120            .execute_query_task(base_task.clone())
8121            .await
8122            .expect_err("unknown query");
8123        assert_eq!(unknown.reason, "rejected_unknown_query");
8124
8125        let mut malformed = base_task;
8126        malformed.query_name = "current".to_string();
8127        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
8128        let malformed = worker
8129            .execute_query_task(malformed)
8130            .await
8131            .expect_err("malformed payload");
8132        assert_eq!(malformed.reason, "query_payload_decode_failed");
8133
8134        let client = Client::new("http://127.0.0.1:8080").expect("client");
8135        let mut unavailable_worker = Worker::new(client, "rust-workers");
8136        unavailable_worker
8137            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8138        let unavailable_task: QueryTask = serde_json::from_value(json!({
8139            "query_task_id": "query-unavailable",
8140            "workflow_type": "counter",
8141            "query_name": "current",
8142            "payload_codec": "json",
8143            "workflow_arguments": {"codec": "json", "blob": "[]"},
8144            "query_arguments": {"codec": "json", "blob": "[]"}
8145        }))
8146        .expect("query task");
8147        let unavailable = unavailable_worker
8148            .execute_query_task(unavailable_task)
8149            .await
8150            .expect_err("query handler unavailable");
8151        assert_eq!(unavailable.reason, "query_handler_unavailable");
8152    }
8153
8154    #[tokio::test]
8155    async fn client_query_decodes_result_and_typed_failure() {
8156        let server = MockWorkerServer::start();
8157        let client = Client::builder(server.base_url())
8158            .timeout(Duration::from_secs(2))
8159            .build()
8160            .expect("client");
8161
8162        let result = client
8163            .query_workflow("counter-1", "current", json!([]))
8164            .await
8165            .expect("query result");
8166        assert_eq!(result, json!({"count": 8}));
8167
8168        let error = client
8169            .query_workflow("counter-1", "missing", json!([]))
8170            .await
8171            .expect_err("unknown query");
8172        let Error::QueryFailed(failure) = error else {
8173            panic!("expected typed query failure");
8174        };
8175        assert_eq!(failure.status, 404);
8176        assert_eq!(failure.reason, "rejected_unknown_query");
8177    }
8178
8179    #[tokio::test]
8180    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
8181        let server = MockWorkerServer::start();
8182        let client = Client::builder(server.base_url())
8183            .timeout(Duration::from_secs(2))
8184            .build()
8185            .expect("client");
8186
8187        let options = WorkflowCommandOptions::new()
8188            .reason("cleanup requested")
8189            .request_id("cancel-17");
8190        let cancelled = client
8191            .cancel_workflow("wf-lifecycle", options)
8192            .await
8193            .expect("instance cancellation");
8194        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
8195        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
8196        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
8197        assert_eq!(
8198            server.request_body("/api/workflows/wf-lifecycle/cancel"),
8199            json!({"reason":"cleanup requested","request_id":"cancel-17"})
8200        );
8201
8202        let terminated = client
8203            .terminate_workflow(
8204                "wf-lifecycle",
8205                WorkflowCommandOptions::new().reason("forced stop"),
8206            )
8207            .await
8208            .expect("instance termination");
8209        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
8210        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
8211
8212        client
8213            .cancel_workflow_run(
8214                "wf-lifecycle",
8215                "run-current",
8216                WorkflowCommandOptions::default(),
8217            )
8218            .await
8219            .expect("selected run cancellation");
8220        client
8221            .terminate_workflow_run(
8222                "wf-lifecycle",
8223                "run-current",
8224                WorkflowCommandOptions::default(),
8225            )
8226            .await
8227            .expect("selected run termination");
8228
8229        for (command, error) in [
8230            (
8231                WorkflowCommandKind::Cancel,
8232                client
8233                    .cancel_workflow_run(
8234                        "wf-lifecycle",
8235                        "run-stale",
8236                        WorkflowCommandOptions::default(),
8237                    )
8238                    .await
8239                    .expect_err("stale cancellation must be rejected"),
8240            ),
8241            (
8242                WorkflowCommandKind::Terminate,
8243                client
8244                    .terminate_workflow_run(
8245                        "wf-lifecycle",
8246                        "run-stale",
8247                        WorkflowCommandOptions::default(),
8248                    )
8249                    .await
8250                    .expect_err("stale termination must be rejected"),
8251            ),
8252        ] {
8253            let Error::WorkflowCommandRejected(rejection) = error else {
8254                panic!("expected typed command rejection");
8255            };
8256            assert_eq!(rejection.command, command);
8257            assert_eq!(rejection.status, 409);
8258            assert_eq!(rejection.reason, "historical_run_command_rejected");
8259            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
8260            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
8261        }
8262    }
8263
8264    #[tokio::test]
8265    async fn workflow_result_returns_each_typed_terminal_outcome() {
8266        let server = MockWorkerServer::start();
8267        let client = Client::builder(server.base_url())
8268            .timeout(Duration::from_secs(2))
8269            .build()
8270            .expect("client");
8271        let options = WorkflowResultOptions {
8272            poll_interval: Duration::ZERO,
8273            timeout: Duration::from_secs(1),
8274        };
8275
8276        let failed = WorkflowHandle {
8277            client: client.clone(),
8278            workflow_id: "wf-failed".to_string(),
8279            run_id: Some("run-failed".to_string()),
8280            workflow_type: "failure".to_string(),
8281        }
8282        .result(options)
8283        .await
8284        .expect_err("failed outcome");
8285        let Error::WorkflowFailed(failure) = failed else {
8286            panic!("expected WorkflowFailed");
8287        };
8288        assert_eq!(failure.workflow_id, "wf-failed");
8289        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
8290        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
8291        assert_eq!(failure.failure_category.as_deref(), Some("application"));
8292        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
8293        assert_eq!(
8294            failure.exception_class.as_deref(),
8295            Some("billing::PaymentError")
8296        );
8297        assert_eq!(failure.non_retryable, Some(true));
8298
8299        for (workflow_id, expected_kind, expected_reason) in [
8300            (
8301                "wf-cancelled",
8302                WorkflowTerminalKind::Cancelled,
8303                "cleanup requested",
8304            ),
8305            (
8306                "wf-terminated",
8307                WorkflowTerminalKind::Terminated,
8308                "forced stop",
8309            ),
8310            (
8311                "wf-timed-out",
8312                WorkflowTerminalKind::TimedOut,
8313                "run_timeout",
8314            ),
8315        ] {
8316            let error = WorkflowHandle {
8317                client: client.clone(),
8318                workflow_id: workflow_id.to_string(),
8319                run_id: None,
8320                workflow_type: "terminal".to_string(),
8321            }
8322            .result(options)
8323            .await
8324            .expect_err("typed terminal outcome");
8325            let outcome = match error {
8326                Error::WorkflowCancelled(outcome) => outcome,
8327                Error::WorkflowTerminated(outcome) => outcome,
8328                Error::WorkflowTimedOut(outcome) => outcome,
8329                other => panic!("unexpected terminal error: {other}"),
8330            };
8331            assert_eq!(outcome.kind, expected_kind);
8332            assert_eq!(outcome.workflow_id, workflow_id);
8333            assert_eq!(outcome.reason, expected_reason);
8334        }
8335
8336        let wait_timeout = WorkflowHandle {
8337            client,
8338            workflow_id: "wf-waiting".to_string(),
8339            run_id: Some("run-waiting".to_string()),
8340            workflow_type: "waiting".to_string(),
8341        }
8342        .result(WorkflowResultOptions {
8343            poll_interval: Duration::ZERO,
8344            timeout: Duration::ZERO,
8345        })
8346        .await
8347        .expect_err("client wait timeout");
8348        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
8349            panic!("expected typed client timeout");
8350        };
8351        assert_eq!(timeout.reason, "result_wait_timeout");
8352        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
8353        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
8354    }
8355
8356    #[tokio::test]
8357    async fn workflow_result_awaits_the_handle_selected_run() {
8358        let server = MockWorkerServer::start();
8359        let client = Client::builder(server.base_url())
8360            .timeout(Duration::from_secs(2))
8361            .build()
8362            .expect("client");
8363
8364        let error = WorkflowHandle {
8365            client,
8366            workflow_id: "wf-selected".to_string(),
8367            run_id: Some("run-selected".to_string()),
8368            workflow_type: "selected".to_string(),
8369        }
8370        .result(WorkflowResultOptions {
8371            poll_interval: Duration::ZERO,
8372            timeout: Duration::from_secs(1),
8373        })
8374        .await
8375        .expect_err("the selected run is cancelled even though the current run completed");
8376
8377        let Error::WorkflowCancelled(outcome) = error else {
8378            panic!("expected selected run cancellation");
8379        };
8380        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
8381        assert_eq!(outcome.reason, "selected run cancelled");
8382        assert_eq!(
8383            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
8384            1
8385        );
8386        assert_eq!(server.request_count("/api/workflows/wf-selected"), 0);
8387    }
8388
8389    #[tokio::test]
8390    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
8391        let server = MockWorkerServer::draining_polls();
8392        let client = Client::builder(server.base_url())
8393            .timeout(Duration::from_secs(2))
8394            .build()
8395            .expect("client");
8396
8397        let workflow = client
8398            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
8399            .await
8400            .expect("workflow drain response");
8401        let activity = client
8402            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
8403            .await
8404            .expect("activity drain response");
8405        let query = client
8406            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
8407            .await
8408            .expect("query drain response");
8409
8410        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
8411            assert_eq!(
8412                outcome,
8413                WorkerPollOutcome::Stop {
8414                    poll_status: Some("draining".to_string()),
8415                    reason: Some("worker_draining".to_string()),
8416                }
8417            );
8418        }
8419
8420        assert!(client
8421            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
8422            .await
8423            .expect("compatibility poll")
8424            .is_none());
8425    }
8426
8427    #[tokio::test]
8428    async fn managed_worker_honors_drain_stop_for_every_task_family() {
8429        let server = MockWorkerServer::draining_polls();
8430        let client = Client::builder(server.base_url())
8431            .timeout(Duration::from_secs(2))
8432            .build()
8433            .expect("client");
8434
8435        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
8436            .worker_id("draining-workflow-worker")
8437            .poll_timeout(Duration::ZERO);
8438        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
8439        workflow_worker
8440            .run()
8441            .await
8442            .expect("workflow drain is a clean stop");
8443
8444        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
8445            .worker_id("draining-activity-worker")
8446            .poll_timeout(Duration::ZERO);
8447        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
8448        activity_worker
8449            .run()
8450            .await
8451            .expect("activity drain is a clean stop");
8452
8453        let mut query_worker = Worker::new(client, "rust-workers")
8454            .worker_id("draining-query-worker")
8455            .poll_timeout(Duration::ZERO);
8456        query_worker.register_query("counter", "current", |_ctx, _args| async {
8457            Ok(Value::Null)
8458        });
8459        query_worker
8460            .run()
8461            .await
8462            .expect("query drain is a clean stop");
8463    }
8464
8465    #[tokio::test]
8466    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
8467        let server = MockWorkerServer::start();
8468        let client = Client::builder(server.base_url())
8469            .timeout(Duration::from_secs(2))
8470            .build()
8471            .expect("client");
8472
8473        let heartbeat = client
8474            .heartbeat_activity_task(
8475                "activity-cancel",
8476                "attempt-cancel",
8477                "rust-worker",
8478                json!({"stage":"cleanup"}),
8479            )
8480            .await
8481            .expect("cancellation heartbeat");
8482        assert!(heartbeat.cancel_requested);
8483        assert!(heartbeat.should_stop());
8484        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
8485        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
8486
8487        let error = client
8488            .complete_activity_task(
8489                "activity-cancel",
8490                "attempt-cancel",
8491                "rust-worker",
8492                json!({"late":true}),
8493                JSON_CODEC,
8494            )
8495            .await
8496            .expect_err("late completion must be refused");
8497        assert!(activity_task_rejection_is_final(&error));
8498        let Error::ActivityTaskRejected(rejection) = error else {
8499            panic!("expected typed activity rejection");
8500        };
8501        assert_eq!(rejection.status, 409);
8502        assert_eq!(rejection.reason, "run_cancelled");
8503        assert!(rejection.cancel_requested);
8504        assert_eq!(rejection.can_continue, Some(false));
8505    }
8506
8507    #[tokio::test]
8508    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
8509        let server = MockWorkerServer::cancelled_activity();
8510        let client = Client::builder(server.base_url())
8511            .timeout(Duration::from_secs(2))
8512            .build()
8513            .expect("client");
8514        let cancellation_observed = Arc::new(AtomicBool::new(false));
8515        let observed = Arc::clone(&cancellation_observed);
8516        let mut worker = Worker::new(client.clone(), "rust-workers")
8517            .worker_id("rust-cancel-worker")
8518            .poll_timeout(Duration::from_millis(10));
8519        worker.register_activity("cancel-aware", move |ctx, _args| {
8520            let observed = Arc::clone(&observed);
8521            async move {
8522                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
8523                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
8524                Ok(json!({"late":"completion"}))
8525            }
8526        });
8527
8528        assert_eq!(
8529            worker.run_once().await.expect("cancelled attempt handled"),
8530            1
8531        );
8532        assert!(cancellation_observed.load(Ordering::SeqCst));
8533        assert_eq!(
8534            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
8535            1
8536        );
8537
8538        let mut restarted = Worker::new(client, "rust-workers")
8539            .worker_id("rust-cancel-worker-restarted")
8540            .poll_timeout(Duration::from_millis(10));
8541        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
8542        assert_eq!(
8543            restarted
8544                .run_once()
8545                .await
8546                .expect("replacement worker continues polling"),
8547            0
8548        );
8549    }
8550
8551    #[tokio::test]
8552    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
8553        let server = MockWorkerServer::start();
8554        let client = Client::builder(server.base_url())
8555            .timeout(Duration::from_secs(2))
8556            .build()
8557            .expect("client");
8558
8559        client
8560            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
8561            .await
8562            .expect("register");
8563        client
8564            .heartbeat_worker("capture-worker", 1, 1)
8565            .await
8566            .expect("heartbeat");
8567        client
8568            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
8569            .await
8570            .expect("workflow poll");
8571        client
8572            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
8573            .await
8574            .expect("activity poll");
8575
8576        for path in [
8577            "/api/worker/register",
8578            "/api/worker/heartbeat",
8579            "/api/worker/workflow-tasks/poll",
8580            "/api/worker/activity-tasks/poll",
8581        ] {
8582            assert_eq!(
8583                server.worker_protocol_for(path).as_deref(),
8584                Some(WORKER_PROTOCOL_VERSION),
8585                "unexpected protocol for {path}"
8586            );
8587        }
8588
8589        assert_eq!(
8590            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
8591            1
8592        );
8593        assert_eq!(
8594            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
8595            1
8596        );
8597    }
8598
8599    #[tokio::test]
8600    async fn query_task_endpoints_send_the_query_feature_protocol() {
8601        let server = MockWorkerServer::start();
8602        let client = Client::builder(server.base_url())
8603            .timeout(Duration::from_secs(2))
8604            .build()
8605            .expect("client");
8606
8607        client
8608            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8609            .await
8610            .expect("query poll");
8611        client
8612            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
8613            .await
8614            .expect("query complete");
8615        client
8616            .fail_query_task(
8617                "query-capture",
8618                "capture-worker",
8619                1,
8620                "failed",
8621                "query_rejected",
8622                "QueryFailed",
8623            )
8624            .await
8625            .expect("query fail");
8626
8627        for path in [
8628            "/api/worker/query-tasks/poll",
8629            "/api/worker/query-tasks/query-capture/complete",
8630            "/api/worker/query-tasks/query-capture/fail",
8631        ] {
8632            assert_eq!(
8633                server.worker_protocol_for(path).as_deref(),
8634                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
8635                "unexpected protocol for {path}"
8636            );
8637        }
8638
8639        assert_eq!(
8640            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
8641            1
8642        );
8643    }
8644
8645    #[tokio::test]
8646    async fn query_protocol_rejection_from_older_server_is_typed() {
8647        let server = MockWorkerServer::reject_query_protocol();
8648        let client = Client::builder(server.base_url())
8649            .timeout(Duration::from_secs(2))
8650            .build()
8651            .expect("client");
8652
8653        let error = client
8654            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8655            .await
8656            .expect_err("server below query protocol floor must reject");
8657        let Error::Protocol(failure) = error else {
8658            panic!("expected typed protocol failure");
8659        };
8660
8661        assert_eq!(failure.status, 400);
8662        assert_eq!(failure.reason, "unsupported_protocol_version");
8663        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
8664        assert_eq!(
8665            failure.requested_version.as_deref(),
8666            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8667        );
8668        assert_eq!(
8669            server
8670                .worker_protocol_for("/api/worker/query-tasks/poll")
8671                .as_deref(),
8672            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8673        );
8674    }
8675
8676    #[tokio::test]
8677    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
8678        let server = MockWorkerServer::reject_query_protocol();
8679        let client = Client::builder(server.base_url())
8680            .timeout(Duration::from_secs(2))
8681            .build()
8682            .expect("client");
8683        let mut worker = Worker::new(client, "rust-workers")
8684            .worker_id("baseline-worker")
8685            .poll_timeout(Duration::from_millis(10));
8686
8687        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
8688            Ok(Value::Null)
8689        });
8690
8691        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
8692        assert_eq!(
8693            server
8694                .worker_protocol_for("/api/worker/workflow-tasks/poll")
8695                .as_deref(),
8696            Some(WORKER_PROTOCOL_VERSION)
8697        );
8698        assert_eq!(
8699            server.worker_protocol_for("/api/worker/query-tasks/poll"),
8700            None,
8701            "a worker without query handlers must not use the query-task endpoint"
8702        );
8703    }
8704
8705    #[tokio::test]
8706    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
8707        let server = MockWorkerServer::reject_query_completion();
8708        let client = Client::builder(server.base_url())
8709            .timeout(Duration::from_secs(2))
8710            .build()
8711            .expect("client");
8712
8713        let error = client
8714            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
8715            .await
8716            .expect_err("expired completion must be rejected");
8717        let Error::QueryFailed(failure) = error else {
8718            panic!("expected typed query failure");
8719        };
8720        assert_eq!(failure.status, 409);
8721        assert_eq!(failure.reason, "query_task_timed_out");
8722
8723        let mut worker = Worker::new(client, "rust-workers")
8724            .worker_id("late-worker")
8725            .poll_timeout(Duration::from_millis(10));
8726        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8727        worker.register_query(
8728            "counter",
8729            "current",
8730            |_ctx, _args| async move { Ok(json!(8)) },
8731        );
8732
8733        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
8734        assert_eq!(
8735            worker
8736                .run_once()
8737                .await
8738                .expect("worker continues after late completion"),
8739            0
8740        );
8741        assert_eq!(
8742            server.request_count("/api/worker/query-tasks/query-late/complete"),
8743            2
8744        );
8745        assert_eq!(
8746            server.request_count("/api/worker/query-tasks/query-late/fail"),
8747            0,
8748            "a server completion rejection must not be reported as an encoding failure"
8749        );
8750    }
8751
8752    #[tokio::test]
8753    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
8754        let server = MockWorkerServer::start();
8755        let client = Client::builder(server.base_url())
8756            .timeout(Duration::from_secs(2))
8757            .build()
8758            .expect("client");
8759        let mut worker = Worker::new(client, "rust-workers")
8760            .worker_id("activity-only-worker")
8761            .poll_timeout(Duration::from_millis(10));
8762
8763        worker.register_activity(
8764            "activity.only",
8765            |_ctx, _args| async move { Ok(Value::Null) },
8766        );
8767
8768        worker.run_until(async {}).await.expect("run worker");
8769    }
8770
8771    #[tokio::test]
8772    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
8773        let server = MockWorkerServer::start();
8774        let client = Client::builder(server.base_url())
8775            .timeout(Duration::from_secs(2))
8776            .build()
8777            .expect("client");
8778        let mut worker = Worker::new(client, "rust-workers")
8779            .worker_id("workflow-only-worker")
8780            .poll_timeout(Duration::from_millis(10));
8781
8782        worker.register_workflow(
8783            "workflow.only",
8784            |_ctx, _input| async move { Ok(Value::Null) },
8785        );
8786
8787        worker.run_until(async {}).await.expect("run worker");
8788    }
8789
8790    #[tokio::test]
8791    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
8792        let server = MockWorkerServer::start();
8793        let client = Client::builder(server.base_url())
8794            .timeout(Duration::from_secs(2))
8795            .build()
8796            .expect("client");
8797        let observations = Arc::new(Mutex::new(Vec::new()));
8798        let observed = Arc::clone(&observations);
8799        let mut worker = Worker::new(client, "rust-workers")
8800            .worker_id("observed-heartbeat-worker")
8801            .poll_timeout(Duration::from_millis(10))
8802            .on_worker_heartbeat(move |observation| {
8803                observed
8804                    .lock()
8805                    .expect("heartbeat observations")
8806                    .push(observation.clone());
8807            });
8808
8809        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
8810            Ok(Value::Null)
8811        });
8812        worker
8813            .run_until(tokio::time::sleep(Duration::from_millis(20)))
8814            .await
8815            .expect("run worker");
8816
8817        let observations = observations.lock().expect("heartbeat observations");
8818        let first = observations.first().expect("heartbeat acknowledgement");
8819        assert_eq!(first.worker_id, "observed-heartbeat-worker");
8820        assert_eq!(first.task_queue, "rust-workers");
8821        assert!(first.acknowledged_at_unix_millis > 0);
8822        assert_eq!(first.acknowledgement, json!({}));
8823    }
8824
8825    #[tokio::test]
8826    async fn query_enabled_worker_stays_live_when_signal_replay_emits_no_commands() {
8827        let server = MockWorkerServer::waiting_query_worker();
8828        let client = Client::builder(server.base_url())
8829            .timeout(Duration::from_secs(2))
8830            .build()
8831            .expect("client");
8832        let observations = Arc::new(Mutex::new(Vec::new()));
8833        let observed = Arc::clone(&observations);
8834        let mut worker = Worker::new(client, "rust-snapshot-workers")
8835            .worker_id("rust-snapshot-worker")
8836            .poll_timeout(Duration::from_millis(10))
8837            .on_worker_heartbeat(move |observation| {
8838                observed
8839                    .lock()
8840                    .expect("heartbeat observations")
8841                    .push(observation.clone());
8842            });
8843
8844        worker.register_workflow("snapshot", |ctx, _input| async move {
8845            ctx.wait_signal("finish").await?;
8846            Ok(json!({"status": "finished"}))
8847        });
8848        worker.register_query("snapshot", "current", |ctx, _args| async move {
8849            let current = ctx
8850                .signals("increment")
8851                .iter()
8852                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
8853                .sum::<i64>();
8854            Ok(json!(current))
8855        });
8856
8857        worker
8858            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
8859            .await
8860            .expect("pending workflow and query poller must remain live until shutdown");
8861
8862        assert!(
8863            observations.lock().expect("heartbeat observations").len() >= 4,
8864            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
8865        );
8866        assert!(
8867            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
8868            "workflow polling must continue after empty replay acknowledgements"
8869        );
8870        assert!(
8871            server.request_count("/api/worker/query-tasks/poll") >= 2,
8872            "query polling must continue after serving the current query"
8873        );
8874        assert_eq!(
8875            server.request_body("/api/worker/register")["capabilities"],
8876            json!([QUERY_TASKS_CAPABILITY])
8877        );
8878
8879        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
8880            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
8881            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
8882            let failure = server.request_body(&fail_path);
8883            assert_eq!(
8884                failure["failure"]["type"],
8885                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
8886            );
8887            assert_eq!(server.request_count(&completion_path), 0);
8888        }
8889
8890        let query_completion =
8891            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
8892        assert_eq!(query_completion["result"], json!(8));
8893    }
8894
8895    #[tokio::test]
8896    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
8897        let server = MockWorkerServer::transient_worker_failures();
8898        let client = Client::builder(server.base_url())
8899            .timeout(Duration::from_secs(2))
8900            .build()
8901            .expect("client");
8902        let mut worker = Worker::new(client, "rust-workers")
8903            .worker_id("retry-worker")
8904            .poll_timeout(Duration::from_millis(10))
8905            .retry_policy(WorkerRetryPolicy {
8906                max_retries: 2,
8907                initial_backoff: Duration::from_millis(1),
8908                max_backoff: Duration::from_millis(1),
8909            });
8910        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8911        worker.register_activity(
8912            "counter.activity",
8913            |_ctx, _input| async move { Ok(Value::Null) },
8914        );
8915        worker.register_query(
8916            "counter",
8917            "current",
8918            |_ctx, _args| async move { Ok(json!(8)) },
8919        );
8920
8921        worker
8922            .run_until(tokio::time::sleep(Duration::from_millis(75)))
8923            .await
8924            .expect("transient failures must not stop the worker");
8925
8926        for path in [
8927            "/api/worker/heartbeat",
8928            "/api/worker/workflow-tasks/poll",
8929            "/api/worker/activity-tasks/poll",
8930            "/api/worker/query-tasks/poll",
8931        ] {
8932            assert!(
8933                server.request_count(path) >= 2,
8934                "{path} must continue after its transient failure"
8935            );
8936        }
8937    }
8938
8939    #[tokio::test]
8940    async fn worker_bounds_transport_retries() {
8941        let server = MockWorkerServer::unavailable_polls();
8942        let client = Client::builder(server.base_url())
8943            .timeout(Duration::from_secs(2))
8944            .build()
8945            .expect("client");
8946        let mut worker = Worker::new(client, "rust-workers")
8947            .worker_id("bounded-retry-worker")
8948            .poll_timeout(Duration::from_millis(10))
8949            .retry_policy(WorkerRetryPolicy {
8950                max_retries: 2,
8951                initial_backoff: Duration::from_millis(1),
8952                max_backoff: Duration::from_millis(1),
8953            });
8954        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8955
8956        let error = worker.run().await.expect_err("retry bound must terminate");
8957        assert!(matches!(error, Error::Transport(_)));
8958        assert_eq!(
8959            server.request_count("/api/worker/workflow-tasks/poll"),
8960            3,
8961            "one initial request plus two retries"
8962        );
8963    }
8964
8965    #[tokio::test]
8966    async fn worker_does_not_retry_authentication_failures() {
8967        let server = MockWorkerServer::unauthorized_polls();
8968        let client = Client::builder(server.base_url())
8969            .timeout(Duration::from_secs(2))
8970            .build()
8971            .expect("client");
8972        let mut worker = Worker::new(client, "rust-workers")
8973            .worker_id("unauthorized-worker")
8974            .poll_timeout(Duration::from_millis(10));
8975        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8976
8977        let error = worker
8978            .run()
8979            .await
8980            .expect_err("authentication must terminate");
8981        let Error::Http { status, body } = error else {
8982            panic!("expected stable HTTP authentication error");
8983        };
8984        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
8985        assert!(body.contains("authentication_failed"));
8986        assert_eq!(
8987            server.request_count("/api/worker/workflow-tasks/poll"),
8988            1,
8989            "authentication failures must not be retried"
8990        );
8991    }
8992
8993    #[derive(Clone, Debug)]
8994    struct CapturedRequest {
8995        path: String,
8996        worker_protocol: Option<String>,
8997        body: String,
8998    }
8999
9000    struct MockWorkerServer {
9001        addr: SocketAddr,
9002        stop: Arc<AtomicBool>,
9003        requests: Arc<Mutex<Vec<CapturedRequest>>>,
9004        thread: Option<thread::JoinHandle<()>>,
9005    }
9006
9007    #[derive(Clone, Copy, Default)]
9008    struct MockWorkerBehavior {
9009        reject_query_protocol: bool,
9010        reject_query_completion: bool,
9011        waiting_query_worker: bool,
9012        poll_failures_per_path: usize,
9013        heartbeat_failures: usize,
9014        unauthorized_polls: bool,
9015        cancelled_activity: bool,
9016        draining_polls: bool,
9017    }
9018
9019    impl MockWorkerServer {
9020        fn start() -> Self {
9021            Self::start_with_behavior(MockWorkerBehavior::default())
9022        }
9023
9024        fn reject_query_protocol() -> Self {
9025            Self::start_with_behavior(MockWorkerBehavior {
9026                reject_query_protocol: true,
9027                ..MockWorkerBehavior::default()
9028            })
9029        }
9030
9031        fn reject_query_completion() -> Self {
9032            Self::start_with_behavior(MockWorkerBehavior {
9033                reject_query_completion: true,
9034                ..MockWorkerBehavior::default()
9035            })
9036        }
9037
9038        fn waiting_query_worker() -> Self {
9039            Self::start_with_behavior(MockWorkerBehavior {
9040                waiting_query_worker: true,
9041                ..MockWorkerBehavior::default()
9042            })
9043        }
9044
9045        fn transient_worker_failures() -> Self {
9046            Self::start_with_behavior(MockWorkerBehavior {
9047                poll_failures_per_path: 1,
9048                heartbeat_failures: 1,
9049                ..MockWorkerBehavior::default()
9050            })
9051        }
9052
9053        fn unavailable_polls() -> Self {
9054            Self::start_with_behavior(MockWorkerBehavior {
9055                poll_failures_per_path: usize::MAX,
9056                ..MockWorkerBehavior::default()
9057            })
9058        }
9059
9060        fn unauthorized_polls() -> Self {
9061            Self::start_with_behavior(MockWorkerBehavior {
9062                unauthorized_polls: true,
9063                ..MockWorkerBehavior::default()
9064            })
9065        }
9066
9067        fn cancelled_activity() -> Self {
9068            Self::start_with_behavior(MockWorkerBehavior {
9069                cancelled_activity: true,
9070                ..MockWorkerBehavior::default()
9071            })
9072        }
9073
9074        fn draining_polls() -> Self {
9075            Self::start_with_behavior(MockWorkerBehavior {
9076                draining_polls: true,
9077                ..MockWorkerBehavior::default()
9078            })
9079        }
9080
9081        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
9082            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
9083            listener
9084                .set_nonblocking(true)
9085                .expect("configure mock listener");
9086            let addr = listener.local_addr().expect("mock server address");
9087            let stop = Arc::new(AtomicBool::new(false));
9088            let server_stop = Arc::clone(&stop);
9089            let requests = Arc::new(Mutex::new(Vec::new()));
9090            let server_requests = Arc::clone(&requests);
9091            let thread = thread::spawn(move || {
9092                while !server_stop.load(Ordering::SeqCst) {
9093                    match listener.accept() {
9094                        Ok((mut stream, _)) => {
9095                            handle_mock_worker_request(&mut stream, &server_requests, behavior)
9096                        }
9097                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
9098                            thread::sleep(Duration::from_millis(5));
9099                        }
9100                        Err(_) => break,
9101                    }
9102                }
9103            });
9104
9105            Self {
9106                addr,
9107                stop,
9108                requests,
9109                thread: Some(thread),
9110            }
9111        }
9112
9113        fn base_url(&self) -> String {
9114            format!("http://{}", self.addr)
9115        }
9116
9117        fn worker_protocol_for(&self, path: &str) -> Option<String> {
9118            self.requests
9119                .lock()
9120                .expect("captured requests")
9121                .iter()
9122                .find(|request| request.path == path)
9123                .and_then(|request| request.worker_protocol.clone())
9124        }
9125
9126        fn request_count(&self, path: &str) -> usize {
9127            self.requests
9128                .lock()
9129                .expect("captured requests")
9130                .iter()
9131                .filter(|request| request.path == path)
9132                .count()
9133        }
9134
9135        fn request_body(&self, path: &str) -> Value {
9136            let requests = self.requests.lock().expect("captured requests");
9137            let body = &requests
9138                .iter()
9139                .find(|request| request.path == path)
9140                .unwrap_or_else(|| panic!("missing request for {path}"))
9141                .body;
9142            serde_json::from_str(body).unwrap_or_else(|error| {
9143                panic!("invalid JSON request body for {path}: {error}: {body:?}")
9144            })
9145        }
9146    }
9147
9148    impl Drop for MockWorkerServer {
9149        fn drop(&mut self) {
9150            self.stop.store(true, Ordering::SeqCst);
9151            let _ = TcpStream::connect(self.addr);
9152
9153            if let Some(thread) = self.thread.take() {
9154                thread.join().expect("join mock server");
9155            }
9156        }
9157    }
9158
9159    fn handle_mock_worker_request(
9160        stream: &mut TcpStream,
9161        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
9162        behavior: MockWorkerBehavior,
9163    ) {
9164        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
9165        let mut buffer = [0_u8; 8192];
9166        let mut request = Vec::new();
9167
9168        loop {
9169            match stream.read(&mut buffer) {
9170                Ok(0) => break,
9171                Ok(read) => {
9172                    request.extend_from_slice(&buffer[..read]);
9173                    if mock_request_is_complete(&request) {
9174                        break;
9175                    }
9176                }
9177                Err(error)
9178                    if matches!(
9179                        error.kind(),
9180                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
9181                    ) =>
9182                {
9183                    break;
9184                }
9185                Err(_) => return,
9186            }
9187        }
9188
9189        let request = String::from_utf8_lossy(&request);
9190        let body = request
9191            .split_once("\r\n\r\n")
9192            .map(|(_, body)| body)
9193            .unwrap_or_default();
9194        let path = request
9195            .lines()
9196            .next()
9197            .and_then(|line| line.split_whitespace().nth(1))
9198            .unwrap_or_default();
9199        let worker_protocol = request.lines().find_map(|line| {
9200            let (name, value) = line.split_once(':')?;
9201            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
9202                .then(|| value.trim().to_string())
9203        });
9204        let request_number = {
9205            let mut requests = requests.lock().expect("captured requests");
9206            requests.push(CapturedRequest {
9207                path: path.to_string(),
9208                worker_protocol: worker_protocol.clone(),
9209                body: body.to_string(),
9210            });
9211            requests
9212                .iter()
9213                .filter(|request| request.path == path)
9214                .count()
9215        };
9216
9217        let is_poll = matches!(
9218            path,
9219            "/api/worker/workflow-tasks/poll"
9220                | "/api/worker/activity-tasks/poll"
9221                | "/api/worker/query-tasks/poll"
9222        );
9223        if is_poll && request_number <= behavior.poll_failures_per_path {
9224            return;
9225        }
9226        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
9227            return;
9228        }
9229        if behavior.unauthorized_polls && is_poll {
9230            write_mock_response(
9231                stream,
9232                "401 Unauthorized",
9233                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
9234            );
9235            return;
9236        }
9237        if behavior.draining_polls && is_poll {
9238            write_mock_response(
9239                stream,
9240                "409 Conflict",
9241                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
9242            );
9243            return;
9244        }
9245
9246        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
9247            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
9248            let body = format!(
9249                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
9250            );
9251            write_mock_response(stream, "400 Bad Request", &body);
9252            return;
9253        }
9254
9255        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
9256        {
9257            write_mock_response(
9258                stream,
9259                "409 Conflict",
9260                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
9261            );
9262            return;
9263        }
9264
9265        if behavior.waiting_query_worker {
9266            if path == "/api/worker/workflow-tasks/poll" && request_number <= 2 {
9267                let amounts = if request_number == 1 {
9268                    vec![3]
9269                } else {
9270                    vec![3, 5]
9271                };
9272                let task_id = if request_number == 1 {
9273                    "snapshot-wait-3"
9274                } else {
9275                    "snapshot-wait-5"
9276                };
9277                let history_events = std::iter::once(json!({
9278                    "event_type": "SignalWaitOpened",
9279                    "payload": {"sequence": 1, "signal_name": "finish"}
9280                }))
9281                .chain(amounts.iter().enumerate().map(|(index, amount)| {
9282                    json!({
9283                        "event_type": "SignalReceived",
9284                        "payload": {
9285                            "signal_id": format!("increment-{amount}"),
9286                            "signal_name": "increment",
9287                            "workflow_sequence": index + 2,
9288                            "payload_codec": DEFAULT_CODEC,
9289                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9290                                .expect("Avro signal envelope")
9291                        }
9292                    })
9293                }))
9294                .collect::<Vec<_>>();
9295                let body = json!({
9296                    "task": {
9297                        "task_id": task_id,
9298                        "workflow_id": "snapshot-1",
9299                        "run_id": "snapshot-run-1",
9300                        "workflow_type": "snapshot",
9301                        "payload_codec": DEFAULT_CODEC,
9302                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9303                            .expect("Avro workflow arguments"),
9304                        "history_events": history_events,
9305                        "workflow_task_attempt": 1,
9306                        "workflow_signal_id": format!("increment-{}", amounts.last().expect("amount")),
9307                        "signal_name": "increment",
9308                        "signal_arguments": encode_value_envelope(
9309                            &json!([amounts.last().expect("amount")]),
9310                            DEFAULT_CODEC,
9311                        )
9312                        .expect("Avro resume signal"),
9313                        "lease_owner": "rust-snapshot-worker"
9314                    }
9315                })
9316                .to_string();
9317                write_mock_response(stream, "200 OK", &body);
9318                return;
9319            }
9320
9321            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
9322                let history_events = [3, 5]
9323                    .into_iter()
9324                    .enumerate()
9325                    .map(|(index, amount)| {
9326                        json!({
9327                            "event_type": "SignalReceived",
9328                            "payload": {
9329                                "signal_id": format!("increment-{amount}"),
9330                                "signal_name": "increment",
9331                                "workflow_sequence": index + 2,
9332                                "payload_codec": DEFAULT_CODEC,
9333                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9334                                    .expect("Avro query signal envelope")
9335                            }
9336                        })
9337                    })
9338                    .collect::<Vec<_>>();
9339                let body = json!({
9340                    "task": {
9341                        "query_task_id": "snapshot-current",
9342                        "query_task_attempt": 1,
9343                        "lease_owner": "rust-snapshot-worker",
9344                        "workflow_id": "snapshot-1",
9345                        "run_id": "snapshot-run-1",
9346                        "workflow_type": "snapshot",
9347                        "query_name": "current",
9348                        "payload_codec": DEFAULT_CODEC,
9349                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9350                            .expect("Avro workflow arguments"),
9351                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9352                            .expect("Avro query arguments"),
9353                        "history_events": history_events,
9354                        "run_status": "waiting"
9355                    }
9356                })
9357                .to_string();
9358                write_mock_response(stream, "200 OK", &body);
9359                return;
9360            }
9361
9362            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
9363                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
9364            {
9365                write_mock_response(
9366                    stream,
9367                    "200 OK",
9368                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
9369                );
9370                return;
9371            }
9372
9373            if path == "/api/worker/query-tasks/snapshot-current/complete" {
9374                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
9375                return;
9376            }
9377        }
9378
9379        let (status, body) = match path {
9380            "/api/worker/register" if behavior.waiting_query_worker => (
9381                "200 OK",
9382                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
9383            ),
9384            "/api/worker/register" => (
9385                "200 OK",
9386                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
9387            ),
9388            "/api/worker/heartbeat" => ("200 OK", "{}"),
9389            "/api/worker/activity-tasks/poll"
9390                if behavior.cancelled_activity && request_number == 1 =>
9391            {
9392                (
9393                    "200 OK",
9394                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
9395                )
9396            }
9397            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
9398                ("200 OK", r#"{"task":null}"#)
9399            }
9400            "/api/worker/query-tasks/poll"
9401                if behavior.reject_query_completion && request_number == 1 =>
9402            {
9403                (
9404                    "200 OK",
9405                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
9406                )
9407            }
9408            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
9409            "/api/worker/query-tasks/query-capture/complete"
9410            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
9411            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
9412                "200 OK",
9413                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
9414            ),
9415            "/api/worker/activity-tasks/activity-cancel/complete" => (
9416                "409 Conflict",
9417                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
9418            ),
9419            "/api/workflows/counter-1/query/current" => (
9420                "200 OK",
9421                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
9422            ),
9423            "/api/workflows/counter-1/query/missing" => (
9424                "404 Not Found",
9425                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
9426            ),
9427            "/api/workflows/wf-lifecycle/cancel" => (
9428                "200 OK",
9429                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
9430            ),
9431            "/api/workflows/wf-lifecycle/terminate" => (
9432                "200 OK",
9433                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
9434            ),
9435            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
9436                "200 OK",
9437                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
9438            ),
9439            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
9440                "200 OK",
9441                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
9442            ),
9443            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
9444            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
9445                "409 Conflict",
9446                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
9447            ),
9448            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
9449                "200 OK",
9450                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
9451            ),
9452            "/api/workflows/wf-cancelled" => (
9453                "200 OK",
9454                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
9455            ),
9456            "/api/workflows/wf-terminated" => (
9457                "200 OK",
9458                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
9459            ),
9460            "/api/workflows/wf-timed-out" => (
9461                "200 OK",
9462                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
9463            ),
9464            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
9465                "200 OK",
9466                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
9467            ),
9468            "/api/workflows/wf-selected" => (
9469                "200 OK",
9470                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
9471            ),
9472            "/api/workflows/wf-selected/runs/run-selected" => (
9473                "200 OK",
9474                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
9475            ),
9476            _ => ("404 Not Found", r#"{"message":"not found"}"#),
9477        };
9478        write_mock_response(stream, status, body);
9479    }
9480
9481    fn mock_request_is_complete(request: &[u8]) -> bool {
9482        let Some(header_end) = request
9483            .windows(4)
9484            .position(|window| window == b"\r\n\r\n")
9485            .map(|position| position + 4)
9486        else {
9487            return false;
9488        };
9489        let headers = String::from_utf8_lossy(&request[..header_end]);
9490        let content_length = headers.lines().find_map(|line| {
9491            let (name, value) = line.split_once(':')?;
9492            name.eq_ignore_ascii_case("content-length")
9493                .then(|| value.trim().parse::<usize>().ok())
9494                .flatten()
9495        });
9496
9497        request.len() >= header_end + content_length.unwrap_or(0)
9498    }
9499
9500    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
9501        let response = format!(
9502            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
9503            body.len()
9504        );
9505
9506        let _ = stream.write_all(response.as_bytes());
9507        let _ = stream.flush();
9508    }
9509}