Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    pin::Pin,
8    sync::{
9        atomic::{AtomicBool, Ordering},
10        Arc, Mutex, OnceLock,
11    },
12    task::{Context as TaskContext, Poll},
13    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
14};
15
16use apache_avro::{from_avro_datum, from_value, to_avro_datum, to_value, Schema};
17use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
18use futures_util::{future::OptionFuture, task::noop_waker_ref};
19use serde::{de::DeserializeOwned, Deserialize, Serialize};
20pub use serde_json::{json, Value};
21use thiserror::Error;
22
23pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
24pub const CONTROL_PLANE_VERSION: &str = "2";
25pub const DEFAULT_CODEC: &str = "avro";
26pub const JSON_CODEC: &str = "json";
27pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
28/// Worker-registration capability for server-routed read-only queries.
29pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
30/// First additive worker protocol that defines query-task transport.
31pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
32
33const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
34const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
35    "Workflow task waiting for scheduled history.";
36const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
37
38const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
39    "lease_expired",
40    "query_task_not_found",
41    "query_task_not_leased",
42    "query_task_timed_out",
43];
44
45const AVRO_PAYLOAD_SCHEMA_JSON: &str = r#"{"type":"record","name":"Payload","namespace":"durable_workflow","fields":[{"name":"json","type":"string"},{"name":"version","type":"int","default":1}]}"#;
46const AVRO_PAYLOAD_VERSION: i32 = 1;
47
48static AVRO_PAYLOAD_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
49
50#[derive(Clone, Copy)]
51enum RequestProtocol {
52    ControlPlane,
53    Worker(&'static str),
54}
55
56impl RequestProtocol {
57    fn is_worker(self) -> bool {
58        matches!(self, Self::Worker(_))
59    }
60}
61
62pub type Result<T> = std::result::Result<T, Error>;
63
64#[derive(Debug, Error)]
65pub enum Error {
66    #[error("transport error: {0}")]
67    Transport(#[from] reqwest::Error),
68    #[error("json error: {0}")]
69    Json(#[from] serde_json::Error),
70    #[error("http {status}: {body}")]
71    Http {
72        status: reqwest::StatusCode,
73        body: String,
74    },
75    #[error("codec error: {0}")]
76    Codec(String),
77    #[error(transparent)]
78    QueryFailed(QueryFailure),
79    #[error(transparent)]
80    Protocol(ProtocolFailure),
81    #[error(transparent)]
82    NonDeterministicReplay(ReplayFailure),
83    #[error(transparent)]
84    ChildWorkflowFailed(ChildWorkflowFailure),
85    #[error(transparent)]
86    ActivityFailed(ActivityFailure),
87    #[error(transparent)]
88    WorkflowCommandRejected(WorkflowCommandRejection),
89    #[error(transparent)]
90    WorkflowFailed(WorkflowTerminalOutcome),
91    #[error(transparent)]
92    WorkflowCancelled(WorkflowTerminalOutcome),
93    #[error(transparent)]
94    WorkflowTerminated(WorkflowTerminalOutcome),
95    #[error(transparent)]
96    WorkflowTimedOut(WorkflowTerminalOutcome),
97    #[error(transparent)]
98    ActivityTaskRejected(ActivityTaskRejection),
99    #[error("workflow handler {0:?} is not registered")]
100    WorkflowNotRegistered(String),
101    #[error("activity handler {0:?} is not registered")]
102    ActivityNotRegistered(String),
103    #[error("workflow future yielded without emitting a durable command")]
104    WorkflowYieldedWithoutCommand,
105    #[error("workflow state lock is poisoned")]
106    WorkflowStatePoisoned,
107    #[error("timer duration is too large for the worker protocol")]
108    TimerDurationOverflow,
109    #[error("operation timed out")]
110    Timeout,
111    #[error("worker loop error: {0}")]
112    WorkerLoop(String),
113    #[error("invalid child workflow options: {0}")]
114    InvalidChildWorkflowOptions(String),
115    #[error(transparent)]
116    InvalidActivityOptions(ActivityOptionsError),
117}
118
119/// The lifecycle command sent to a workflow execution.
120#[derive(Clone, Copy, Debug, PartialEq, Eq)]
121pub enum WorkflowCommandKind {
122    Cancel,
123    Terminate,
124}
125
126impl WorkflowCommandKind {
127    fn as_str(self) -> &'static str {
128        match self {
129            Self::Cancel => "cancel",
130            Self::Terminate => "terminate",
131        }
132    }
133}
134
135/// Optional structured fields for a cancellation or termination request.
136#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
137pub struct WorkflowCommandOptions {
138    #[serde(skip_serializing_if = "Option::is_none")]
139    pub reason: Option<String>,
140    #[serde(skip_serializing_if = "Option::is_none")]
141    pub request_id: Option<String>,
142}
143
144/// Server-enforced timeout policy for a workflow start.
145///
146/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
147/// only bounds how long the caller waits. A server deadline produces a terminal
148/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
149/// `run_timeout`.
150#[derive(Clone, Debug, PartialEq, Eq)]
151pub struct WorkflowStartOptions {
152    pub execution_timeout_seconds: u64,
153    pub run_timeout_seconds: u64,
154}
155
156impl Default for WorkflowStartOptions {
157    fn default() -> Self {
158        Self {
159            execution_timeout_seconds: 3600,
160            run_timeout_seconds: 600,
161        }
162    }
163}
164
165impl WorkflowStartOptions {
166    pub fn new() -> Self {
167        Self::default()
168    }
169
170    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
171        self.execution_timeout_seconds = seconds;
172        self
173    }
174
175    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
176        self.run_timeout_seconds = seconds;
177        self
178    }
179
180    fn validate(&self) -> Result<()> {
181        if self.execution_timeout_seconds == 0 {
182            return Err(Error::Codec(
183                "execution_timeout_seconds must be at least 1".to_string(),
184            ));
185        }
186        if self.run_timeout_seconds == 0 {
187            return Err(Error::Codec(
188                "run_timeout_seconds must be at least 1".to_string(),
189            ));
190        }
191        if self.run_timeout_seconds > self.execution_timeout_seconds {
192            return Err(Error::Codec(
193                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
194            ));
195        }
196
197        Ok(())
198    }
199}
200
201impl WorkflowCommandOptions {
202    pub fn new() -> Self {
203        Self::default()
204    }
205
206    pub fn reason(mut self, reason: impl Into<String>) -> Self {
207        self.reason = Some(reason.into());
208        self
209    }
210
211    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
212        self.request_id = Some(request_id.into());
213        self
214    }
215}
216
217/// The accepted, machine-readable result of a lifecycle command.
218#[derive(Clone, Debug, PartialEq)]
219pub struct WorkflowCommandResult {
220    pub command: WorkflowCommandKind,
221    pub workflow_id: String,
222    pub run_id: Option<String>,
223    pub outcome: Option<String>,
224    pub reason: Option<String>,
225    pub command_status: Option<String>,
226    pub raw: Value,
227}
228
229/// A stable rejection returned by instance- or selected-run lifecycle commands.
230#[derive(Clone, Debug, Error)]
231#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
232pub struct WorkflowCommandRejection {
233    pub command: WorkflowCommandKind,
234    pub status: u16,
235    pub reason: String,
236    pub message: String,
237    pub workflow_id: String,
238    pub run_id: Option<String>,
239    pub target_scope: Option<String>,
240    pub body: Value,
241}
242
243/// Stable terminal categories returned by [`WorkflowHandle::result`].
244#[derive(Clone, Copy, Debug, PartialEq, Eq)]
245pub enum WorkflowTerminalKind {
246    Failed,
247    Cancelled,
248    Terminated,
249    TimedOut,
250}
251
252/// A typed terminal workflow outcome with durable identity and failure metadata.
253///
254/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
255/// of parsing its display representation. Fields remain `None` when an older
256/// server did not publish that metadata.
257#[derive(Clone, Debug, Error)]
258#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
259pub struct WorkflowTerminalOutcome {
260    pub kind: WorkflowTerminalKind,
261    pub workflow_id: String,
262    pub run_id: Option<String>,
263    pub reason: String,
264    pub failure_category: Option<String>,
265    pub failure_id: Option<String>,
266    pub exception_type: Option<String>,
267    pub exception_class: Option<String>,
268    pub non_retryable: Option<bool>,
269    pub message: Option<String>,
270    pub exception: Option<Value>,
271    pub raw: Value,
272}
273
274/// A worker-side activity settlement or heartbeat rejected by durable state.
275#[derive(Clone, Debug, Error)]
276#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
277pub struct ActivityTaskRejection {
278    pub operation: String,
279    pub status: u16,
280    pub reason: String,
281    pub task_id: String,
282    pub activity_attempt_id: String,
283    pub cancel_requested: bool,
284    pub can_continue: Option<bool>,
285    pub run_closed_reason: Option<String>,
286    pub body: Value,
287}
288
289/// Stable validation categories for [`ActivityOptions`].
290#[derive(Clone, Copy, Debug, PartialEq, Eq)]
291pub enum ActivityOptionsErrorKind {
292    EmptyTaskQueue,
293    EmptyRetryPolicy,
294    InvalidMaxAttempts,
295    BackoffWithoutRetryBudget,
296    TooManyBackoffIntervals,
297    InvalidBackoffCoefficient,
298    BackoffGenerationTooLarge,
299    BackoffOverflow,
300    EmptyNonRetryableErrorType,
301    TimeoutNotPositive,
302    TimeoutOverflow,
303    TimeoutOrder,
304}
305
306/// A machine-readable activity-options validation failure.
307#[derive(Clone, Debug, Error, PartialEq, Eq)]
308#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
309pub struct ActivityOptionsError {
310    pub kind: ActivityOptionsErrorKind,
311    pub field: Option<&'static str>,
312    pub message: String,
313}
314
315impl ActivityOptionsError {
316    fn new(
317        kind: ActivityOptionsErrorKind,
318        field: Option<&'static str>,
319        message: impl Into<String>,
320    ) -> Self {
321        Self {
322            kind,
323            field,
324            message: message.into(),
325        }
326    }
327}
328
329/// Stable terminal categories returned when an awaited activity does not succeed.
330#[derive(Clone, Copy, Debug, PartialEq, Eq)]
331pub enum ActivityFailureKind {
332    Failed,
333    Cancelled,
334    TimedOut,
335}
336
337/// A stable, machine-readable terminal activity failure.
338///
339/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
340/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
341#[derive(Clone, Debug, Error)]
342#[error("activity failed ({reason}): {message}")]
343pub struct ActivityFailure {
344    pub kind: ActivityFailureKind,
345    pub reason: String,
346    pub message: String,
347    pub activity_execution_id: Option<String>,
348    pub activity_attempt_id: Option<String>,
349    pub activity_type: Option<String>,
350    pub activity_class: Option<String>,
351    pub attempt_number: Option<u64>,
352    pub failure_id: Option<String>,
353    pub failure_category: Option<String>,
354    pub timeout_kind: Option<String>,
355    pub non_retryable: bool,
356    pub exception_type: Option<String>,
357    pub exception_class: Option<String>,
358    pub code: Option<Value>,
359    pub exception: Option<Value>,
360}
361
362/// Stable terminal categories returned when an awaited child does not succeed.
363#[derive(Clone, Copy, Debug, PartialEq, Eq)]
364pub enum ChildWorkflowFailureKind {
365    Failed,
366    Cancelled,
367    Terminated,
368}
369
370/// A stable, machine-readable child workflow failure delivered to its parent.
371///
372/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
373/// of parsing the display message. Child and parent identifiers retain the
374/// relationship recorded in durable history across worker restarts.
375#[derive(Clone, Debug, Error)]
376#[error("child workflow failed ({reason}): {message}")]
377pub struct ChildWorkflowFailure {
378    pub kind: ChildWorkflowFailureKind,
379    pub reason: String,
380    pub message: String,
381    pub parent_workflow_id: Option<String>,
382    pub parent_workflow_run_id: Option<String>,
383    pub child_workflow_id: Option<String>,
384    pub child_workflow_run_id: Option<String>,
385    pub child_workflow_type: Option<String>,
386    pub failure_id: Option<String>,
387    pub failure_category: Option<String>,
388    pub exception_type: Option<String>,
389    pub exception_class: Option<String>,
390    pub non_retryable: bool,
391    pub code: Option<Value>,
392    pub exception: Option<Value>,
393}
394
395/// The identity of one durable workflow execution.
396#[derive(Clone, Debug, PartialEq, Eq)]
397pub struct WorkflowIdentity {
398    pub workflow_id: Option<String>,
399    pub run_id: Option<String>,
400}
401
402/// A successful child result together with its durable parent-child identity.
403#[derive(Clone, Debug, PartialEq)]
404pub struct ChildWorkflowResult {
405    pub parent: WorkflowIdentity,
406    pub child: WorkflowIdentity,
407    pub child_workflow_type: Option<String>,
408    pub result: Value,
409}
410
411/// Server behavior when a parent closes while its child is still open.
412#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
413pub enum ParentClosePolicy {
414    #[default]
415    Abandon,
416    RequestCancel,
417    Terminate,
418}
419
420impl ParentClosePolicy {
421    fn as_str(self) -> &'static str {
422        match self {
423            Self::Abandon => "abandon",
424            Self::RequestCancel => "request_cancel",
425            Self::Terminate => "terminate",
426        }
427    }
428}
429
430/// Durable retry policy for one child workflow invocation.
431#[derive(Clone, Debug, Default, PartialEq, Eq)]
432pub struct ChildWorkflowRetryPolicy {
433    pub max_attempts: Option<u32>,
434    pub backoff_seconds: Vec<u64>,
435    pub non_retryable_error_types: Vec<String>,
436}
437
438/// Options recorded with a child-workflow command.
439///
440/// The task queue is mandatory so routing is explicit and replay-stable.
441#[derive(Clone, Debug, PartialEq, Eq)]
442pub struct ChildWorkflowOptions {
443    pub task_queue: String,
444    pub parent_close_policy: ParentClosePolicy,
445    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
446    pub execution_timeout_seconds: Option<u64>,
447    pub run_timeout_seconds: Option<u64>,
448}
449
450impl ChildWorkflowOptions {
451    pub fn new(task_queue: impl Into<String>) -> Self {
452        Self {
453            task_queue: task_queue.into(),
454            parent_close_policy: ParentClosePolicy::Abandon,
455            retry_policy: None,
456            execution_timeout_seconds: None,
457            run_timeout_seconds: None,
458        }
459    }
460
461    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
462        self.parent_close_policy = policy;
463        self
464    }
465
466    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
467        self.retry_policy = Some(policy);
468        self
469    }
470
471    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
472        self.execution_timeout_seconds = Some(seconds);
473        self
474    }
475
476    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
477        self.run_timeout_seconds = Some(seconds);
478        self
479    }
480}
481
482/// Backoff intervals for one durable activity retry policy.
483#[derive(Clone, Debug, PartialEq, Eq)]
484pub enum ActivityBackoff {
485    /// Use these intervals between attempts. The server repeats the final
486    /// interval if the retry budget contains more attempts than entries.
487    Explicit(Vec<Duration>),
488    /// Generate one interval for every retry using integer exponential growth.
489    Exponential {
490        initial_interval: Duration,
491        coefficient: u32,
492        maximum_interval: Option<Duration>,
493    },
494}
495
496/// Durable server-side retry policy for one activity execution.
497#[derive(Clone, Debug, Default, PartialEq, Eq)]
498pub struct ActivityRetryPolicy {
499    pub max_attempts: Option<u32>,
500    pub backoff: Option<ActivityBackoff>,
501    pub non_retryable_error_types: Vec<String>,
502}
503
504impl ActivityRetryPolicy {
505    /// Start a policy with a finite attempt budget, including the first attempt.
506    pub fn new(max_attempts: u32) -> Self {
507        Self {
508            max_attempts: Some(max_attempts),
509            ..Self::default()
510        }
511    }
512
513    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
514        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
515        self
516    }
517
518    pub fn exponential_backoff(
519        mut self,
520        initial_interval: Duration,
521        coefficient: u32,
522        maximum_interval: Option<Duration>,
523    ) -> Self {
524        self.backoff = Some(ActivityBackoff::Exponential {
525            initial_interval,
526            coefficient,
527            maximum_interval,
528        });
529        self
530    }
531
532    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
533        self.non_retryable_error_types.push(error_type.into());
534        self
535    }
536
537    pub fn non_retryable_error_types(
538        mut self,
539        error_types: impl IntoIterator<Item = impl Into<String>>,
540    ) -> Self {
541        self.non_retryable_error_types
542            .extend(error_types.into_iter().map(Into::into));
543        self
544    }
545}
546
547/// Options recorded atomically on one deterministic `schedule_activity` command.
548///
549/// Durations are rounded up to whole seconds when encoded, so the server never
550/// receives a shorter timeout or backoff than the caller requested.
551#[derive(Clone, Debug, Default, PartialEq, Eq)]
552pub struct ActivityOptions {
553    pub task_queue: Option<String>,
554    pub retry_policy: Option<ActivityRetryPolicy>,
555    pub start_to_close_timeout: Option<Duration>,
556    pub schedule_to_start_timeout: Option<Duration>,
557    pub schedule_to_close_timeout: Option<Duration>,
558    pub heartbeat_timeout: Option<Duration>,
559}
560
561impl ActivityOptions {
562    pub fn new() -> Self {
563        Self::default()
564    }
565
566    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
567        self.task_queue = Some(task_queue.into());
568        self
569    }
570
571    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
572        self.retry_policy = Some(policy);
573        self
574    }
575
576    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
577        self.start_to_close_timeout = Some(timeout);
578        self
579    }
580
581    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
582        self.schedule_to_start_timeout = Some(timeout);
583        self
584    }
585
586    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
587        self.schedule_to_close_timeout = Some(timeout);
588        self
589    }
590
591    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
592        self.heartbeat_timeout = Some(timeout);
593        self
594    }
595
596    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
597        if self
598            .task_queue
599            .as_deref()
600            .is_some_and(|queue| queue.trim().is_empty())
601        {
602            return Err(ActivityOptionsError::new(
603                ActivityOptionsErrorKind::EmptyTaskQueue,
604                Some("task_queue"),
605                "task_queue must not be empty",
606            ));
607        }
608
609        for (field, value) in [
610            ("start_to_close_timeout", self.start_to_close_timeout),
611            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
612            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
613            ("heartbeat_timeout", self.heartbeat_timeout),
614        ] {
615            if value.is_some_and(|value| value.is_zero()) {
616                return Err(ActivityOptionsError::new(
617                    ActivityOptionsErrorKind::TimeoutNotPositive,
618                    Some(field),
619                    format!("{field} must be positive"),
620                ));
621            }
622        }
623
624        validate_timeout_order(
625            "heartbeat_timeout",
626            self.heartbeat_timeout,
627            "start_to_close_timeout",
628            self.start_to_close_timeout,
629        )?;
630        validate_timeout_order(
631            "start_to_close_timeout",
632            self.start_to_close_timeout,
633            "schedule_to_close_timeout",
634            self.schedule_to_close_timeout,
635        )?;
636        validate_timeout_order(
637            "schedule_to_start_timeout",
638            self.schedule_to_start_timeout,
639            "schedule_to_close_timeout",
640            self.schedule_to_close_timeout,
641        )?;
642
643        Ok(ValidatedActivityOptions {
644            task_queue: self.task_queue.clone(),
645            retry_policy: self
646                .retry_policy
647                .as_ref()
648                .map(validate_activity_retry_policy)
649                .transpose()?,
650            start_to_close_timeout: timeout_seconds(
651                "start_to_close_timeout",
652                self.start_to_close_timeout,
653            )?,
654            schedule_to_start_timeout: timeout_seconds(
655                "schedule_to_start_timeout",
656                self.schedule_to_start_timeout,
657            )?,
658            schedule_to_close_timeout: timeout_seconds(
659                "schedule_to_close_timeout",
660                self.schedule_to_close_timeout,
661            )?,
662            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
663        })
664    }
665}
666
667#[derive(Clone, Debug)]
668struct ValidatedActivityOptions {
669    task_queue: Option<String>,
670    retry_policy: Option<Value>,
671    start_to_close_timeout: Option<u64>,
672    schedule_to_start_timeout: Option<u64>,
673    schedule_to_close_timeout: Option<u64>,
674    heartbeat_timeout: Option<u64>,
675}
676
677fn validate_timeout_order(
678    smaller_name: &'static str,
679    smaller: Option<Duration>,
680    larger_name: &'static str,
681    larger: Option<Duration>,
682) -> std::result::Result<(), ActivityOptionsError> {
683    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
684        return Err(ActivityOptionsError::new(
685            ActivityOptionsErrorKind::TimeoutOrder,
686            Some(smaller_name),
687            format!("{smaller_name} must be <= {larger_name}"),
688        ));
689    }
690    Ok(())
691}
692
693fn timeout_seconds(
694    field: &'static str,
695    value: Option<Duration>,
696) -> std::result::Result<Option<u64>, ActivityOptionsError> {
697    value
698        .map(|value| {
699            activity_protocol_seconds(value).ok_or_else(|| {
700                ActivityOptionsError::new(
701                    ActivityOptionsErrorKind::TimeoutOverflow,
702                    Some(field),
703                    format!("{field} is too large for the worker protocol"),
704                )
705            })
706        })
707        .transpose()
708}
709
710fn duration_seconds_ceil(value: Duration) -> Option<u64> {
711    value
712        .as_secs()
713        .checked_add(u64::from(value.subsec_nanos() > 0))
714}
715
716fn activity_protocol_seconds(value: Duration) -> Option<u64> {
717    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
718}
719
720fn validate_activity_retry_policy(
721    policy: &ActivityRetryPolicy,
722) -> std::result::Result<Value, ActivityOptionsError> {
723    if policy.max_attempts.is_none()
724        && policy.backoff.is_none()
725        && policy.non_retryable_error_types.is_empty()
726    {
727        return Err(ActivityOptionsError::new(
728            ActivityOptionsErrorKind::EmptyRetryPolicy,
729            Some("retry_policy"),
730            "retry_policy must configure at least one field",
731        ));
732    }
733    if policy.max_attempts == Some(0) {
734        return Err(ActivityOptionsError::new(
735            ActivityOptionsErrorKind::InvalidMaxAttempts,
736            Some("retry_policy.max_attempts"),
737            "max_attempts must be >= 1",
738        ));
739    }
740    if policy
741        .non_retryable_error_types
742        .iter()
743        .any(|error_type| error_type.trim().is_empty())
744    {
745        return Err(ActivityOptionsError::new(
746            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
747            Some("retry_policy.non_retryable_error_types"),
748            "non_retryable_error_types must not contain empty values",
749        ));
750    }
751
752    let backoff_seconds = match &policy.backoff {
753        None => None,
754        Some(backoff) => {
755            let max_attempts = policy.max_attempts.ok_or_else(|| {
756                ActivityOptionsError::new(
757                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
758                    Some("retry_policy.backoff"),
759                    "backoff requires max_attempts",
760                )
761            })?;
762            let retry_count = max_attempts.saturating_sub(1) as usize;
763            let intervals = match backoff {
764                ActivityBackoff::Explicit(intervals) => {
765                    if intervals.len() > retry_count {
766                        return Err(ActivityOptionsError::new(
767                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
768                            Some("retry_policy.backoff"),
769                            "backoff interval count must not exceed max_attempts - 1",
770                        ));
771                    }
772                    intervals.clone()
773                }
774                ActivityBackoff::Exponential {
775                    initial_interval,
776                    coefficient,
777                    maximum_interval,
778                } => {
779                    if *coefficient < 1 {
780                        return Err(ActivityOptionsError::new(
781                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
782                            Some("retry_policy.backoff.coefficient"),
783                            "backoff coefficient must be >= 1",
784                        ));
785                    }
786                    if retry_count > 10_000 {
787                        return Err(ActivityOptionsError::new(
788                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
789                            Some("retry_policy.max_attempts"),
790                            "generated backoff supports at most 10000 retry intervals",
791                        ));
792                    }
793                    let mut current = *initial_interval;
794                    let mut intervals = Vec::with_capacity(retry_count);
795                    for _ in 0..retry_count {
796                        let interval = maximum_interval
797                            .map(|maximum| current.min(maximum))
798                            .unwrap_or(current);
799                        intervals.push(interval);
800                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
801                            break;
802                        }
803                        current = current.checked_mul(*coefficient).ok_or_else(|| {
804                            ActivityOptionsError::new(
805                                ActivityOptionsErrorKind::BackoffOverflow,
806                                Some("retry_policy.backoff"),
807                                "generated backoff interval overflowed",
808                            )
809                        })?;
810                    }
811                    intervals
812                }
813            };
814            Some(
815                intervals
816                    .into_iter()
817                    .map(|interval| {
818                        activity_protocol_seconds(interval).ok_or_else(|| {
819                            ActivityOptionsError::new(
820                                ActivityOptionsErrorKind::BackoffOverflow,
821                                Some("retry_policy.backoff"),
822                                "backoff interval is too large for the worker protocol",
823                            )
824                        })
825                    })
826                    .collect::<std::result::Result<Vec<_>, _>>()?,
827            )
828        }
829    };
830
831    let mut encoded = serde_json::Map::new();
832    if let Some(max_attempts) = policy.max_attempts {
833        encoded.insert("max_attempts".to_string(), json!(max_attempts));
834    }
835    if let Some(backoff_seconds) = backoff_seconds {
836        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
837    }
838    if !policy.non_retryable_error_types.is_empty() {
839        let mut canonical_error_types = Vec::new();
840        for error_type in policy
841            .non_retryable_error_types
842            .iter()
843            .map(|error_type| error_type.trim())
844        {
845            if !canonical_error_types.contains(&error_type) {
846                canonical_error_types.push(error_type);
847            }
848        }
849        encoded.insert(
850            "non_retryable_error_types".to_string(),
851            json!(canonical_error_types),
852        );
853    }
854    Ok(Value::Object(encoded))
855}
856
857/// A stable, machine-readable failure raised when workflow code no longer
858/// reconstructs the durable command stream recorded in history.
859#[derive(Clone, Debug, Error)]
860#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
861pub struct ReplayFailure {
862    pub reason: String,
863    pub sequence: Option<u64>,
864    pub expected: Option<String>,
865    pub actual: Option<String>,
866    pub message: String,
867}
868
869impl ReplayFailure {
870    fn new(
871        reason: impl Into<String>,
872        sequence: Option<u64>,
873        expected: Option<String>,
874        actual: Option<String>,
875        message: impl Into<String>,
876    ) -> Self {
877        Self {
878            reason: reason.into(),
879            sequence,
880            expected,
881            actual,
882            message: message.into(),
883        }
884    }
885}
886
887/// A stable, machine-readable workflow query or query-task settlement failure.
888#[derive(Clone, Debug, Error)]
889#[error("query failed ({reason}, HTTP {status}): {message}")]
890pub struct QueryFailure {
891    pub status: u16,
892    pub reason: String,
893    pub message: String,
894    pub body: Value,
895}
896
897/// A stable failure returned when a server rejects an SDK protocol version.
898#[derive(Clone, Debug, Error)]
899#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
900pub struct ProtocolFailure {
901    pub status: u16,
902    pub reason: String,
903    pub message: String,
904    pub supported_version: Option<String>,
905    pub requested_version: Option<String>,
906    pub body: Value,
907}
908
909#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
910pub struct PayloadEnvelope {
911    pub codec: String,
912    pub blob: String,
913}
914
915impl PayloadEnvelope {
916    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
917        encode_payload(value, DEFAULT_CODEC)
918    }
919
920    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
921        encode_payload(value, JSON_CODEC)
922    }
923}
924
925pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
926    let value = serde_json::to_value(value)?;
927    let blob = encode_value_blob(&value, codec)?;
928
929    Ok(PayloadEnvelope {
930        codec: codec.to_string(),
931        blob,
932    })
933}
934
935pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
936    let value = decode_blob(&envelope.blob, &envelope.codec)?;
937    Ok(serde_json::from_value(value)?)
938}
939
940fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
941    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
942}
943
944fn encode_value_blob(value: &Value, codec: &str) -> Result<String> {
945    match codec {
946        JSON_CODEC => Ok(serde_json::to_string(value)?),
947        DEFAULT_CODEC => encode_avro_generic(value),
948        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
949    }
950}
951
952fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
953    if value.is_null() {
954        return Ok(Value::Null);
955    }
956
957    if let Some(object) = value.as_object() {
958        if let (Some(codec), Some(blob)) = (
959            object.get("codec").and_then(Value::as_str),
960            object.get("blob").and_then(Value::as_str),
961        ) {
962            return decode_blob(blob, codec);
963        }
964    }
965
966    if let Some(blob) = value.as_str() {
967        return decode_blob(blob, fallback_codec);
968    }
969
970    Ok(value.clone())
971}
972
973fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
974    match codec {
975        JSON_CODEC => Ok(serde_json::from_str(blob)?),
976        DEFAULT_CODEC => decode_avro_generic(blob),
977        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
978    }
979}
980
981fn encode_avro_generic(value: &Value) -> Result<String> {
982    let json = serde_json::to_string(value)?;
983    let datum = to_value(AvroPayload {
984        json,
985        version: AVRO_PAYLOAD_VERSION,
986    })
987    .map_err(|err| Error::Codec(format!("could not convert avro generic wrapper: {err}")))?;
988    let datum = to_avro_datum(avro_payload_schema()?, datum)
989        .map_err(|err| Error::Codec(format!("could not encode avro generic wrapper: {err}")))?;
990
991    let mut bytes = Vec::with_capacity(datum.len() + 1);
992    bytes.push(0x00);
993    bytes.extend_from_slice(&datum);
994    Ok(BASE64.encode(bytes))
995}
996
997fn decode_avro_generic(blob: &str) -> Result<Value> {
998    let bytes = BASE64
999        .decode(blob)
1000        .map_err(|err| Error::Codec(format!("invalid avro base64 payload: {err}")))?;
1001
1002    if bytes.is_empty() {
1003        return Err(Error::Codec("avro payload is empty".to_string()));
1004    }
1005
1006    match bytes[0] {
1007        0x00 => {}
1008        0x01 => {
1009            return Err(Error::Codec(
1010                "typed avro payloads require a schema context; v1 supports the generic wrapper"
1011                    .to_string(),
1012            ));
1013        }
1014        other => {
1015            return Err(Error::Codec(format!(
1016                "unknown avro payload prefix 0x{other:02x}"
1017            )));
1018        }
1019    }
1020
1021    let mut datum = &bytes[1..];
1022    let datum = from_avro_datum(avro_payload_schema()?, &mut datum, None)
1023        .map_err(|err| Error::Codec(format!("could not decode avro generic wrapper: {err}")))?;
1024    let payload: AvroPayload = from_value(&datum)
1025        .map_err(|err| Error::Codec(format!("invalid avro generic wrapper record: {err}")))?;
1026
1027    if payload.version != AVRO_PAYLOAD_VERSION {
1028        return Err(Error::Codec(format!(
1029            "unsupported avro generic wrapper version {}",
1030            payload.version
1031        )));
1032    }
1033
1034    Ok(serde_json::from_str(&payload.json)?)
1035}
1036
1037#[derive(Debug, Serialize, Deserialize)]
1038struct AvroPayload {
1039    json: String,
1040    version: i32,
1041}
1042
1043fn avro_payload_schema() -> Result<&'static Schema> {
1044    match AVRO_PAYLOAD_SCHEMA.get_or_init(|| {
1045        Schema::parse_str(AVRO_PAYLOAD_SCHEMA_JSON)
1046            .map_err(|err| format!("could not parse avro payload schema: {err}"))
1047    }) {
1048        Ok(schema) => Ok(schema),
1049        Err(message) => Err(Error::Codec(message.clone())),
1050    }
1051}
1052
1053#[derive(Clone, Debug)]
1054pub struct Client {
1055    http: reqwest::Client,
1056    base_url: String,
1057    token: Option<String>,
1058    control_token: Option<String>,
1059    worker_token: Option<String>,
1060    namespace: String,
1061}
1062
1063impl Client {
1064    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1065        Self::builder(base_url).build()
1066    }
1067
1068    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1069        ClientBuilder {
1070            base_url: base_url.into(),
1071            token: None,
1072            control_token: None,
1073            worker_token: None,
1074            namespace: "default".to_string(),
1075            timeout: Duration::from_secs(60),
1076        }
1077    }
1078
1079    pub async fn health(&self) -> Result<Value> {
1080        self.request_json(
1081            reqwest::Method::GET,
1082            "/health",
1083            RequestProtocol::ControlPlane,
1084            Option::<&Value>::None,
1085        )
1086        .await
1087    }
1088
1089    pub async fn cluster_info(&self) -> Result<Value> {
1090        self.request_json(
1091            reqwest::Method::GET,
1092            "/cluster/info",
1093            RequestProtocol::ControlPlane,
1094            Option::<&Value>::None,
1095        )
1096        .await
1097    }
1098
1099    pub async fn start_workflow<T: Serialize>(
1100        &self,
1101        workflow_type: &str,
1102        task_queue: &str,
1103        workflow_id: &str,
1104        input: T,
1105    ) -> Result<WorkflowHandle> {
1106        self.start_workflow_with_options(
1107            workflow_type,
1108            task_queue,
1109            workflow_id,
1110            WorkflowStartOptions::default(),
1111            input,
1112        )
1113        .await
1114    }
1115
1116    /// Start a workflow with explicit server-enforced execution and run
1117    /// deadlines.
1118    pub async fn start_workflow_with_options<T: Serialize>(
1119        &self,
1120        workflow_type: &str,
1121        task_queue: &str,
1122        workflow_id: &str,
1123        options: WorkflowStartOptions,
1124        input: T,
1125    ) -> Result<WorkflowHandle> {
1126        options.validate()?;
1127        let input = serde_json::to_value(input)?;
1128        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1129        let body = json!({
1130            "workflow_id": workflow_id,
1131            "workflow_type": workflow_type,
1132            "task_queue": task_queue,
1133            "input": input_envelope,
1134            "execution_timeout_seconds": options.execution_timeout_seconds,
1135            "run_timeout_seconds": options.run_timeout_seconds
1136        });
1137
1138        let data: Value = self
1139            .request_json(
1140                reqwest::Method::POST,
1141                "/workflows",
1142                RequestProtocol::ControlPlane,
1143                Some(&body),
1144            )
1145            .await?;
1146
1147        Ok(WorkflowHandle {
1148            client: self.clone(),
1149            workflow_id: data
1150                .get("workflow_id")
1151                .and_then(Value::as_str)
1152                .unwrap_or(workflow_id)
1153                .to_string(),
1154            run_id: data
1155                .get("run_id")
1156                .and_then(Value::as_str)
1157                .map(str::to_string),
1158            workflow_type: data
1159                .get("workflow_type")
1160                .and_then(Value::as_str)
1161                .unwrap_or(workflow_type)
1162                .to_string(),
1163        })
1164    }
1165
1166    pub async fn signal_workflow<T: Serialize>(
1167        &self,
1168        workflow_id: &str,
1169        signal_name: &str,
1170        input: T,
1171    ) -> Result<Value> {
1172        let input = serde_json::to_value(input)?;
1173        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1174        let body = json!({
1175            "input": input_envelope
1176        });
1177        let path = format!("/workflows/{workflow_id}/signal/{signal_name}");
1178        self.request_json(
1179            reqwest::Method::POST,
1180            &path,
1181            RequestProtocol::ControlPlane,
1182            Some(&body),
1183        )
1184        .await
1185    }
1186
1187    /// Request cooperative cancellation of the current run for an instance.
1188    pub async fn cancel_workflow(
1189        &self,
1190        workflow_id: &str,
1191        options: WorkflowCommandOptions,
1192    ) -> Result<WorkflowCommandResult> {
1193        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1194            .await
1195    }
1196
1197    /// Request cooperative cancellation only if `run_id` is still current.
1198    pub async fn cancel_workflow_run(
1199        &self,
1200        workflow_id: &str,
1201        run_id: &str,
1202        options: WorkflowCommandOptions,
1203    ) -> Result<WorkflowCommandResult> {
1204        self.workflow_command(
1205            workflow_id,
1206            Some(run_id),
1207            WorkflowCommandKind::Cancel,
1208            options,
1209        )
1210        .await
1211    }
1212
1213    /// Forcefully terminate the current run for an instance.
1214    pub async fn terminate_workflow(
1215        &self,
1216        workflow_id: &str,
1217        options: WorkflowCommandOptions,
1218    ) -> Result<WorkflowCommandResult> {
1219        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1220            .await
1221    }
1222
1223    /// Forcefully terminate only if `run_id` is still current.
1224    pub async fn terminate_workflow_run(
1225        &self,
1226        workflow_id: &str,
1227        run_id: &str,
1228        options: WorkflowCommandOptions,
1229    ) -> Result<WorkflowCommandResult> {
1230        self.workflow_command(
1231            workflow_id,
1232            Some(run_id),
1233            WorkflowCommandKind::Terminate,
1234            options,
1235        )
1236        .await
1237    }
1238
1239    async fn workflow_command(
1240        &self,
1241        workflow_id: &str,
1242        run_id: Option<&str>,
1243        command: WorkflowCommandKind,
1244        options: WorkflowCommandOptions,
1245    ) -> Result<WorkflowCommandResult> {
1246        let path = match run_id {
1247            Some(run_id) => format!(
1248                "/workflows/{workflow_id}/runs/{run_id}/{}",
1249                command.as_str()
1250            ),
1251            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1252        };
1253        let data = match self
1254            .request_json(
1255                reqwest::Method::POST,
1256                &path,
1257                RequestProtocol::ControlPlane,
1258                Some(&options),
1259            )
1260            .await
1261        {
1262            Ok(data) => data,
1263            Err(Error::Http { status, body }) => {
1264                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1265                    command,
1266                    status,
1267                    body,
1268                    workflow_id,
1269                    run_id,
1270                )));
1271            }
1272            Err(error) => return Err(error),
1273        };
1274
1275        Ok(workflow_command_result(command, data, workflow_id, run_id))
1276    }
1277
1278    /// Execute a named, read-only query against a running or completed workflow.
1279    ///
1280    /// Arguments and results use the platform payload envelope. Server and
1281    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1282    /// reason, HTTP status, and original response body.
1283    pub async fn query_workflow<T: Serialize>(
1284        &self,
1285        workflow_id: &str,
1286        query_name: &str,
1287        input: T,
1288    ) -> Result<Value> {
1289        let input = serde_json::to_value(input)?;
1290        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1291        let body = json!({
1292            "input": input_envelope
1293        });
1294        let path = format!("/workflows/{workflow_id}/query/{query_name}");
1295        let response: Value = match self
1296            .request_json(
1297                reqwest::Method::POST,
1298                &path,
1299                RequestProtocol::ControlPlane,
1300                Some(&body),
1301            )
1302            .await
1303        {
1304            Ok(response) => response,
1305            Err(Error::Http { status, body }) => {
1306                return Err(Error::QueryFailed(query_failure(status, body)));
1307            }
1308            Err(error) => return Err(error),
1309        };
1310
1311        if let Some(envelope) = response
1312            .get("result_envelope")
1313            .filter(|envelope| !envelope.is_null())
1314        {
1315            return decode_wire_value(envelope, DEFAULT_CODEC);
1316        }
1317
1318        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1319    }
1320
1321    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
1322        let path = format!("/workflows/{workflow_id}");
1323        let mut data: WorkflowDescription = self
1324            .request_json(
1325                reqwest::Method::GET,
1326                &path,
1327                RequestProtocol::ControlPlane,
1328                Option::<&Value>::None,
1329            )
1330            .await?;
1331        data.decode_payloads()?;
1332        Ok(data)
1333    }
1334
1335    /// Describe one selected run, including historical terminal runs.
1336    pub async fn describe_workflow_run(
1337        &self,
1338        workflow_id: &str,
1339        run_id: &str,
1340    ) -> Result<WorkflowDescription> {
1341        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
1342        let mut data: WorkflowDescription = self
1343            .request_json(
1344                reqwest::Method::GET,
1345                &path,
1346                RequestProtocol::ControlPlane,
1347                Option::<&Value>::None,
1348            )
1349            .await?;
1350        data.decode_payloads()?;
1351        Ok(data)
1352    }
1353
1354    pub async fn register_worker(
1355        &self,
1356        worker_id: &str,
1357        task_queue: &str,
1358        supported_workflow_types: Vec<String>,
1359        supported_activity_types: Vec<String>,
1360        max_concurrent_workflow_tasks: usize,
1361        max_concurrent_activity_tasks: usize,
1362    ) -> Result<RegisterWorkerResponse> {
1363        self.register_worker_with_capabilities(
1364            worker_id,
1365            task_queue,
1366            supported_workflow_types,
1367            supported_activity_types,
1368            max_concurrent_workflow_tasks,
1369            max_concurrent_activity_tasks,
1370            Vec::new(),
1371        )
1372        .await
1373    }
1374
1375    /// Register a worker and explicitly advertise additive worker capabilities.
1376    pub async fn register_worker_with_capabilities(
1377        &self,
1378        worker_id: &str,
1379        task_queue: &str,
1380        supported_workflow_types: Vec<String>,
1381        supported_activity_types: Vec<String>,
1382        max_concurrent_workflow_tasks: usize,
1383        max_concurrent_activity_tasks: usize,
1384        capabilities: Vec<String>,
1385    ) -> Result<RegisterWorkerResponse> {
1386        let body = json!({
1387            "worker_id": worker_id,
1388            "task_queue": task_queue,
1389            "runtime": "rust",
1390            "sdk_version": SDK_VERSION,
1391            "supported_workflow_types": supported_workflow_types,
1392            "supported_activity_types": supported_activity_types,
1393            "capabilities": capabilities,
1394            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
1395            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
1396        });
1397
1398        self.request_json(
1399            reqwest::Method::POST,
1400            "/worker/register",
1401            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1402            Some(&body),
1403        )
1404        .await
1405    }
1406
1407    /// Long-poll for an ephemeral, read-only workflow query task.
1408    pub async fn poll_query_task(
1409        &self,
1410        worker_id: &str,
1411        task_queue: &str,
1412        timeout: Duration,
1413    ) -> Result<Option<QueryTask>> {
1414        Ok(self
1415            .poll_query_task_response(worker_id, task_queue, timeout)
1416            .await?
1417            .task)
1418    }
1419
1420    /// Poll a query task while preserving server stop and drain metadata.
1421    pub async fn poll_query_task_response(
1422        &self,
1423        worker_id: &str,
1424        task_queue: &str,
1425        timeout: Duration,
1426    ) -> Result<PollQueryTaskResponse> {
1427        let timeout_seconds = long_poll_timeout_seconds(timeout);
1428        let body = json!({
1429            "worker_id": worker_id,
1430            "task_queue": task_queue,
1431            "poll_request_id": unique_request_id("rust-query-poll"),
1432            "timeout_seconds": timeout_seconds,
1433        });
1434        worker_poll_response(
1435            self.request_json_with_timeout(
1436                reqwest::Method::POST,
1437                "/worker/query-tasks/poll",
1438                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1439                Some(&body),
1440                timeout + Duration::from_secs(5),
1441            )
1442            .await,
1443        )
1444    }
1445
1446    /// Complete a query task without appending workflow history.
1447    pub async fn complete_query_task(
1448        &self,
1449        query_task_id: &str,
1450        lease_owner: &str,
1451        query_task_attempt: u64,
1452        result: Value,
1453        codec: &str,
1454    ) -> Result<Value> {
1455        let result_envelope = encode_value_envelope(&result, codec)?;
1456        self.complete_query_task_with_envelope(
1457            query_task_id,
1458            lease_owner,
1459            query_task_attempt,
1460            result,
1461            result_envelope,
1462        )
1463        .await
1464    }
1465
1466    async fn complete_query_task_with_envelope(
1467        &self,
1468        query_task_id: &str,
1469        lease_owner: &str,
1470        query_task_attempt: u64,
1471        result: Value,
1472        result_envelope: Value,
1473    ) -> Result<Value> {
1474        let body = json!({
1475            "lease_owner": lease_owner,
1476            "query_task_attempt": query_task_attempt,
1477            "result": result,
1478            "result_envelope": result_envelope,
1479        });
1480        let path = format!("/worker/query-tasks/{query_task_id}/complete");
1481        let response = self
1482            .request_json(
1483                reqwest::Method::POST,
1484                &path,
1485                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1486                Some(&body),
1487            )
1488            .await;
1489        query_task_response(response)
1490    }
1491
1492    /// Report a stable machine-readable query-task failure.
1493    pub async fn fail_query_task(
1494        &self,
1495        query_task_id: &str,
1496        lease_owner: &str,
1497        query_task_attempt: u64,
1498        message: impl Into<String>,
1499        reason: impl Into<String>,
1500        failure_type: impl Into<String>,
1501    ) -> Result<Value> {
1502        let body = json!({
1503            "lease_owner": lease_owner,
1504            "query_task_attempt": query_task_attempt,
1505            "failure": {
1506                "message": message.into(),
1507                "reason": reason.into(),
1508                "type": failure_type.into(),
1509            }
1510        });
1511        let path = format!("/worker/query-tasks/{query_task_id}/fail");
1512        let response = self
1513            .request_json(
1514                reqwest::Method::POST,
1515                &path,
1516                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1517                Some(&body),
1518            )
1519            .await;
1520        query_task_response(response)
1521    }
1522
1523    pub async fn heartbeat_worker(
1524        &self,
1525        worker_id: &str,
1526        workflow_available: usize,
1527        activity_available: usize,
1528    ) -> Result<Value> {
1529        let body = json!({
1530            "worker_id": worker_id,
1531            "task_slots": {
1532                "workflow_available": workflow_available,
1533                "activity_available": activity_available
1534            },
1535            "process_metrics": {
1536                "process_id": std::process::id(),
1537                "process_uptime_seconds": 0
1538            }
1539        });
1540
1541        self.request_json(
1542            reqwest::Method::POST,
1543            "/worker/heartbeat",
1544            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1545            Some(&body),
1546        )
1547        .await
1548    }
1549
1550    pub async fn poll_workflow_task(
1551        &self,
1552        worker_id: &str,
1553        task_queue: &str,
1554        timeout: Duration,
1555    ) -> Result<Option<WorkflowTask>> {
1556        Ok(self
1557            .poll_workflow_task_response(worker_id, task_queue, timeout)
1558            .await?
1559            .task)
1560    }
1561
1562    pub async fn poll_workflow_task_response(
1563        &self,
1564        worker_id: &str,
1565        task_queue: &str,
1566        timeout: Duration,
1567    ) -> Result<PollWorkflowTaskResponse> {
1568        let body = json!({
1569            "worker_id": worker_id,
1570            "task_queue": task_queue,
1571            "timeout_seconds": long_poll_timeout_seconds(timeout),
1572        });
1573        let mut data: PollWorkflowTaskResponse = worker_poll_response(
1574            self.request_json_with_timeout(
1575                reqwest::Method::POST,
1576                "/worker/workflow-tasks/poll",
1577                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1578                Some(&body),
1579                timeout + Duration::from_secs(5),
1580            )
1581            .await,
1582        )?;
1583
1584        if let Some(task) = data.task.as_mut() {
1585            self.fetch_remaining_workflow_history(worker_id, task)
1586                .await?;
1587        }
1588
1589        Ok(data)
1590    }
1591
1592    async fn fetch_remaining_workflow_history(
1593        &self,
1594        worker_id: &str,
1595        task: &mut WorkflowTask,
1596    ) -> Result<()> {
1597        let mut next_token = task.next_history_page_token.clone();
1598
1599        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
1600            let lease_owner = task
1601                .lease_owner
1602                .clone()
1603                .unwrap_or_else(|| worker_id.to_string());
1604            let page = self
1605                .workflow_task_history_page(
1606                    &task.task_id,
1607                    &lease_owner,
1608                    task.workflow_task_attempt,
1609                    &token,
1610                )
1611                .await?;
1612
1613            task.append_history_page(page);
1614
1615            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
1616                return Err(Error::Codec(
1617                    "workflow history pagination returned the same page token".to_string(),
1618                ));
1619            }
1620
1621            next_token = task.next_history_page_token.clone();
1622        }
1623
1624        Ok(())
1625    }
1626
1627    async fn workflow_task_history_page(
1628        &self,
1629        task_id: &str,
1630        lease_owner: &str,
1631        workflow_task_attempt: u64,
1632        next_history_page_token: &str,
1633    ) -> Result<WorkflowTaskHistoryPage> {
1634        let body = json!({
1635            "lease_owner": lease_owner,
1636            "workflow_task_attempt": workflow_task_attempt,
1637            "next_history_page_token": next_history_page_token
1638        });
1639        let path = format!("/worker/workflow-tasks/{task_id}/history");
1640
1641        self.request_json(
1642            reqwest::Method::POST,
1643            &path,
1644            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1645            Some(&body),
1646        )
1647        .await
1648    }
1649
1650    pub async fn complete_workflow_task(
1651        &self,
1652        task_id: &str,
1653        lease_owner: &str,
1654        workflow_task_attempt: u64,
1655        commands: Vec<Value>,
1656    ) -> Result<Value> {
1657        let body = json!({
1658            "lease_owner": lease_owner,
1659            "workflow_task_attempt": workflow_task_attempt,
1660            "commands": commands
1661        });
1662        let path = format!("/worker/workflow-tasks/{task_id}/complete");
1663        self.request_json(
1664            reqwest::Method::POST,
1665            &path,
1666            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1667            Some(&body),
1668        )
1669        .await
1670    }
1671
1672    pub async fn fail_workflow_task(
1673        &self,
1674        task_id: &str,
1675        lease_owner: &str,
1676        workflow_task_attempt: u64,
1677        message: impl Into<String>,
1678    ) -> Result<Value> {
1679        self.fail_workflow_task_with_type(
1680            task_id,
1681            lease_owner,
1682            workflow_task_attempt,
1683            message,
1684            "RustWorkflowTaskFailure",
1685        )
1686        .await
1687    }
1688
1689    async fn fail_workflow_task_with_type(
1690        &self,
1691        task_id: &str,
1692        lease_owner: &str,
1693        workflow_task_attempt: u64,
1694        message: impl Into<String>,
1695        failure_type: &str,
1696    ) -> Result<Value> {
1697        let body = json!({
1698            "lease_owner": lease_owner,
1699            "workflow_task_attempt": workflow_task_attempt,
1700            "failure": {
1701                "message": message.into(),
1702                "type": failure_type
1703            }
1704        });
1705        let path = format!("/worker/workflow-tasks/{task_id}/fail");
1706        self.request_json(
1707            reqwest::Method::POST,
1708            &path,
1709            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1710            Some(&body),
1711        )
1712        .await
1713    }
1714
1715    pub async fn poll_activity_task(
1716        &self,
1717        worker_id: &str,
1718        task_queue: &str,
1719        timeout: Duration,
1720    ) -> Result<Option<ActivityTask>> {
1721        Ok(self
1722            .poll_activity_task_response(worker_id, task_queue, timeout)
1723            .await?
1724            .task)
1725    }
1726
1727    /// Poll an activity task while preserving server stop and drain metadata.
1728    pub async fn poll_activity_task_response(
1729        &self,
1730        worker_id: &str,
1731        task_queue: &str,
1732        timeout: Duration,
1733    ) -> Result<PollActivityTaskResponse> {
1734        let body = json!({
1735            "worker_id": worker_id,
1736            "task_queue": task_queue,
1737            "timeout_seconds": long_poll_timeout_seconds(timeout),
1738        });
1739        let data: PollActivityTaskResponse = worker_poll_response(
1740            self.request_json_with_timeout(
1741                reqwest::Method::POST,
1742                "/worker/activity-tasks/poll",
1743                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1744                Some(&body),
1745                timeout + Duration::from_secs(5),
1746            )
1747            .await,
1748        )?;
1749        Ok(data)
1750    }
1751
1752    pub async fn complete_activity_task(
1753        &self,
1754        task_id: &str,
1755        activity_attempt_id: &str,
1756        lease_owner: &str,
1757        result: Value,
1758        codec: &str,
1759    ) -> Result<Value> {
1760        let result = encode_value_envelope(&result, codec)?;
1761        let body = json!({
1762            "activity_attempt_id": activity_attempt_id,
1763            "lease_owner": lease_owner,
1764            "result": result
1765        });
1766        let path = format!("/worker/activity-tasks/{task_id}/complete");
1767        activity_task_response(
1768            self.request_json(
1769                reqwest::Method::POST,
1770                &path,
1771                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1772                Some(&body),
1773            )
1774            .await,
1775            "complete",
1776            task_id,
1777            activity_attempt_id,
1778        )
1779    }
1780
1781    pub async fn fail_activity_task(
1782        &self,
1783        task_id: &str,
1784        activity_attempt_id: &str,
1785        lease_owner: &str,
1786        message: impl Into<String>,
1787        non_retryable: bool,
1788    ) -> Result<Value> {
1789        let body = json!({
1790            "activity_attempt_id": activity_attempt_id,
1791            "lease_owner": lease_owner,
1792            "failure": {
1793                "message": message.into(),
1794                "type": "RustActivityFailure",
1795                "non_retryable": non_retryable
1796            }
1797        });
1798        let path = format!("/worker/activity-tasks/{task_id}/fail");
1799        activity_task_response(
1800            self.request_json(
1801                reqwest::Method::POST,
1802                &path,
1803                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1804                Some(&body),
1805            )
1806            .await,
1807            "fail",
1808            task_id,
1809            activity_attempt_id,
1810        )
1811    }
1812
1813    pub async fn heartbeat_activity_task(
1814        &self,
1815        task_id: &str,
1816        activity_attempt_id: &str,
1817        lease_owner: &str,
1818        details: Value,
1819    ) -> Result<ActivityHeartbeatResponse> {
1820        let body = json!({
1821            "activity_attempt_id": activity_attempt_id,
1822            "lease_owner": lease_owner,
1823            "details": details
1824        });
1825        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
1826        activity_task_response(
1827            self.request_json(
1828                reqwest::Method::POST,
1829                &path,
1830                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1831                Some(&body),
1832            )
1833            .await,
1834            "heartbeat",
1835            task_id,
1836            activity_attempt_id,
1837        )
1838    }
1839
1840    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
1841        &self,
1842        method: reqwest::Method,
1843        path: &str,
1844        protocol: RequestProtocol,
1845        body: Option<&B>,
1846    ) -> Result<T> {
1847        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
1848            .await
1849    }
1850
1851    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
1852        &self,
1853        method: reqwest::Method,
1854        path: &str,
1855        protocol: RequestProtocol,
1856        body: Option<&B>,
1857        timeout: Duration,
1858    ) -> Result<T> {
1859        let mut request = self
1860            .http
1861            .request(method, format!("{}/api{}", self.base_url, path))
1862            .timeout(timeout)
1863            .header(reqwest::header::ACCEPT, "application/json")
1864            .header(reqwest::header::CONTENT_TYPE, "application/json")
1865            .header("X-Namespace", &self.namespace);
1866
1867        match protocol {
1868            RequestProtocol::Worker(version) => {
1869                request = request.header("X-Durable-Workflow-Protocol-Version", version);
1870            }
1871            RequestProtocol::ControlPlane => {
1872                request = request.header(
1873                    "X-Durable-Workflow-Control-Plane-Version",
1874                    CONTROL_PLANE_VERSION,
1875                );
1876            }
1877        }
1878
1879        if let Some(token) = self.auth_token(protocol.is_worker()) {
1880            request = request.bearer_auth(token);
1881        }
1882
1883        if let Some(body) = body {
1884            request = request.json(body);
1885        }
1886
1887        let response = request.send().await?;
1888        let status = response.status();
1889        let bytes = response.bytes().await?;
1890
1891        if !status.is_success() {
1892            let body = String::from_utf8_lossy(&bytes).to_string();
1893            if let Some(protocol) = protocol_failure(status, &body) {
1894                return Err(Error::Protocol(protocol));
1895            }
1896            return Err(Error::Http { status, body });
1897        }
1898
1899        if bytes.is_empty() {
1900            return Ok(serde_json::from_value(Value::Null)?);
1901        }
1902
1903        Ok(serde_json::from_slice(&bytes)?)
1904    }
1905
1906    fn auth_token(&self, worker: bool) -> Option<&str> {
1907        if worker {
1908            self.worker_token
1909                .as_deref()
1910                .or(self.token.as_deref())
1911                .or(self.control_token.as_deref())
1912        } else {
1913            self.control_token
1914                .as_deref()
1915                .or(self.token.as_deref())
1916                .or(self.worker_token.as_deref())
1917        }
1918    }
1919}
1920
1921fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
1922    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1923    let reason = body
1924        .get("reason")
1925        .and_then(Value::as_str)
1926        .unwrap_or("query_rejected")
1927        .to_string();
1928    let message = body
1929        .get("message")
1930        .or_else(|| body.get("error"))
1931        .and_then(Value::as_str)
1932        .unwrap_or("workflow query was rejected")
1933        .to_string();
1934
1935    QueryFailure {
1936        status: status.as_u16(),
1937        reason,
1938        message,
1939        body,
1940    }
1941}
1942
1943fn workflow_command_result(
1944    command: WorkflowCommandKind,
1945    data: Value,
1946    workflow_id: &str,
1947    run_id: Option<&str>,
1948) -> WorkflowCommandResult {
1949    WorkflowCommandResult {
1950        command,
1951        workflow_id: data
1952            .get("workflow_id")
1953            .and_then(Value::as_str)
1954            .unwrap_or(workflow_id)
1955            .to_string(),
1956        run_id: data
1957            .get("run_id")
1958            .and_then(Value::as_str)
1959            .or(run_id)
1960            .map(str::to_string),
1961        outcome: data
1962            .get("outcome")
1963            .and_then(Value::as_str)
1964            .map(str::to_string),
1965        reason: data
1966            .get("reason")
1967            .and_then(Value::as_str)
1968            .map(str::to_string),
1969        command_status: data
1970            .get("command_status")
1971            .and_then(Value::as_str)
1972            .map(str::to_string),
1973        raw: data,
1974    }
1975}
1976
1977fn workflow_command_rejection(
1978    command: WorkflowCommandKind,
1979    status: reqwest::StatusCode,
1980    raw_body: String,
1981    workflow_id: &str,
1982    run_id: Option<&str>,
1983) -> WorkflowCommandRejection {
1984    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1985    WorkflowCommandRejection {
1986        command,
1987        status: status.as_u16(),
1988        reason: body
1989            .get("reason")
1990            .and_then(Value::as_str)
1991            .unwrap_or("workflow_command_rejected")
1992            .to_string(),
1993        message: body
1994            .get("message")
1995            .or_else(|| body.get("error"))
1996            .and_then(Value::as_str)
1997            .unwrap_or("workflow lifecycle command was rejected")
1998            .to_string(),
1999        workflow_id: body
2000            .get("workflow_id")
2001            .and_then(Value::as_str)
2002            .unwrap_or(workflow_id)
2003            .to_string(),
2004        run_id: body
2005            .get("run_id")
2006            .and_then(Value::as_str)
2007            .or(run_id)
2008            .map(str::to_string),
2009        target_scope: body
2010            .get("target_scope")
2011            .and_then(Value::as_str)
2012            .map(str::to_string),
2013        body,
2014    }
2015}
2016
2017fn query_task_response(response: Result<Value>) -> Result<Value> {
2018    match response {
2019        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2020        response => response,
2021    }
2022}
2023
2024fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2025    match response {
2026        Err(Error::Http { status, body })
2027            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2028        {
2029            Ok(serde_json::from_str(&body)?)
2030        }
2031        response => response,
2032    }
2033}
2034
2035fn worker_poll_body_is_stop(body: &str) -> bool {
2036    serde_json::from_str::<Value>(body)
2037        .ok()
2038        .is_some_and(|body| {
2039            worker_poll_is_stop(
2040                body.get("poll_status").and_then(Value::as_str),
2041                body.get("reason").and_then(Value::as_str),
2042            )
2043        })
2044}
2045
2046fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2047    matches!(poll_status, Some("draining" | "stopped"))
2048        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2049}
2050
2051fn query_task_rejection_is_final(error: &Error) -> bool {
2052    matches!(
2053        error,
2054        Error::QueryFailed(failure)
2055            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2056    )
2057}
2058
2059fn activity_task_response<T>(
2060    response: Result<T>,
2061    operation: &str,
2062    task_id: &str,
2063    activity_attempt_id: &str,
2064) -> Result<T> {
2065    match response {
2066        Err(Error::Http { status, body }) => {
2067            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2068            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2069                operation: operation.to_string(),
2070                status: status.as_u16(),
2071                reason: body
2072                    .get("reason")
2073                    .and_then(Value::as_str)
2074                    .unwrap_or("activity_task_rejected")
2075                    .to_string(),
2076                task_id: body
2077                    .get("task_id")
2078                    .and_then(Value::as_str)
2079                    .unwrap_or(task_id)
2080                    .to_string(),
2081                activity_attempt_id: body
2082                    .get("activity_attempt_id")
2083                    .and_then(Value::as_str)
2084                    .unwrap_or(activity_attempt_id)
2085                    .to_string(),
2086                cancel_requested: body
2087                    .get("cancel_requested")
2088                    .and_then(Value::as_bool)
2089                    .unwrap_or(false),
2090                can_continue: body.get("can_continue").and_then(Value::as_bool),
2091                run_closed_reason: body
2092                    .get("run_closed_reason")
2093                    .and_then(Value::as_str)
2094                    .map(str::to_string),
2095                body,
2096            }))
2097        }
2098        response => response,
2099    }
2100}
2101
2102fn activity_task_rejection_is_final(error: &Error) -> bool {
2103    matches!(
2104        error,
2105        Error::ActivityTaskRejected(rejection)
2106            if matches!(
2107                rejection.reason.as_str(),
2108                "run_cancelled"
2109                    | "run_terminated"
2110                    | "attempt_closed"
2111                    | "stale_attempt"
2112                    | "activity_cancelled"
2113                    | "task_cancelled"
2114                    | "run_closed"
2115                    | "activity_not_running"
2116                    | "attempt_not_found"
2117            )
2118    )
2119}
2120
2121fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2122    let body: Value = serde_json::from_str(raw_body).ok()?;
2123    let reason = body.get("reason")?.as_str()?;
2124    if !matches!(
2125        reason,
2126        "missing_protocol_version"
2127            | "unsupported_protocol_version"
2128            | "missing_control_plane_version"
2129            | "unsupported_control_plane_version"
2130    ) {
2131        return None;
2132    }
2133
2134    Some(ProtocolFailure {
2135        status: status.as_u16(),
2136        reason: reason.to_string(),
2137        message: body
2138            .get("message")
2139            .or_else(|| body.get("error"))
2140            .and_then(Value::as_str)
2141            .unwrap_or("protocol version rejected")
2142            .to_string(),
2143        supported_version: body
2144            .get("supported_version")
2145            .and_then(Value::as_str)
2146            .map(str::to_string),
2147        requested_version: body
2148            .get("requested_version")
2149            .and_then(Value::as_str)
2150            .map(str::to_string),
2151        body,
2152    })
2153}
2154
2155fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2156    timeout
2157        .as_secs()
2158        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2159        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2160}
2161
2162fn worker_operation_is_retryable(error: &Error) -> bool {
2163    match error {
2164        Error::Transport(error) => {
2165            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2166        }
2167        Error::Http { status, .. } => {
2168            matches!(
2169                *status,
2170                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2171            ) || status.is_server_error()
2172        }
2173        _ => false,
2174    }
2175}
2176
2177fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
2178    let exponent = retry.saturating_sub(1).min(31) as u32;
2179    policy
2180        .initial_backoff
2181        .saturating_mul(1_u32 << exponent)
2182        .min(policy.max_backoff)
2183}
2184
2185#[derive(Debug)]
2186pub struct ClientBuilder {
2187    base_url: String,
2188    token: Option<String>,
2189    control_token: Option<String>,
2190    worker_token: Option<String>,
2191    namespace: String,
2192    timeout: Duration,
2193}
2194
2195impl ClientBuilder {
2196    pub fn token(mut self, token: Option<String>) -> Self {
2197        self.token = token;
2198        self
2199    }
2200
2201    pub fn control_token(mut self, token: Option<String>) -> Self {
2202        self.control_token = token;
2203        self
2204    }
2205
2206    pub fn worker_token(mut self, token: Option<String>) -> Self {
2207        self.worker_token = token;
2208        self
2209    }
2210
2211    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
2212        self.namespace = namespace.into();
2213        self
2214    }
2215
2216    pub fn timeout(mut self, timeout: Duration) -> Self {
2217        self.timeout = timeout;
2218        self
2219    }
2220
2221    pub fn build(self) -> Result<Client> {
2222        Ok(Client {
2223            http: reqwest::Client::builder().timeout(self.timeout).build()?,
2224            base_url: self.base_url.trim_end_matches('/').to_string(),
2225            token: self.token,
2226            control_token: self.control_token,
2227            worker_token: self.worker_token,
2228            namespace: self.namespace,
2229        })
2230    }
2231}
2232
2233#[derive(Clone, Debug)]
2234pub struct WorkflowHandle {
2235    client: Client,
2236    pub workflow_id: String,
2237    pub run_id: Option<String>,
2238    pub workflow_type: String,
2239}
2240
2241impl WorkflowHandle {
2242    pub async fn describe(&self) -> Result<WorkflowDescription> {
2243        self.client.describe_workflow(&self.workflow_id).await
2244    }
2245
2246    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
2247        self.client
2248            .signal_workflow(&self.workflow_id, signal_name, input)
2249            .await
2250    }
2251
2252    /// Request cooperative cancellation of whichever run is current.
2253    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
2254        self.client
2255            .cancel_workflow(&self.workflow_id, options)
2256            .await
2257    }
2258
2259    /// Request cancellation only if this handle's selected run is still current.
2260    pub async fn cancel_selected_run(
2261        &self,
2262        options: WorkflowCommandOptions,
2263    ) -> Result<WorkflowCommandResult> {
2264        let run_id = self.run_id.as_deref().ok_or_else(|| {
2265            Error::Codec("run_id is required for selected-run cancellation".to_string())
2266        })?;
2267        self.client
2268            .cancel_workflow_run(&self.workflow_id, run_id, options)
2269            .await
2270    }
2271
2272    /// Forcefully terminate whichever run is current.
2273    pub async fn terminate(
2274        &self,
2275        options: WorkflowCommandOptions,
2276    ) -> Result<WorkflowCommandResult> {
2277        self.client
2278            .terminate_workflow(&self.workflow_id, options)
2279            .await
2280    }
2281
2282    /// Terminate only if this handle's selected run is still current.
2283    pub async fn terminate_selected_run(
2284        &self,
2285        options: WorkflowCommandOptions,
2286    ) -> Result<WorkflowCommandResult> {
2287        let run_id = self.run_id.as_deref().ok_or_else(|| {
2288            Error::Codec("run_id is required for selected-run termination".to_string())
2289        })?;
2290        self.client
2291            .terminate_workflow_run(&self.workflow_id, run_id, options)
2292            .await
2293    }
2294
2295    /// Execute a named, read-only query against this workflow.
2296    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
2297        self.client
2298            .query_workflow(&self.workflow_id, query_name, input)
2299            .await
2300    }
2301
2302    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
2303        let started = Instant::now();
2304
2305        loop {
2306            let description = match self.run_id.as_deref() {
2307                Some(run_id) => {
2308                    self.client
2309                        .describe_workflow_run(&self.workflow_id, run_id)
2310                        .await?
2311                }
2312                None => self.describe().await?,
2313            };
2314            if description.is_completed() {
2315                return Ok(description.output.unwrap_or(Value::Null));
2316            }
2317
2318            if description.is_terminal() {
2319                let outcome = workflow_terminal_outcome(
2320                    &description,
2321                    &self.workflow_id,
2322                    self.run_id.as_deref(),
2323                );
2324                return Err(match outcome.kind {
2325                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
2326                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
2327                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
2328                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
2329                });
2330            }
2331
2332            if started.elapsed() >= options.timeout {
2333                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
2334                    kind: WorkflowTerminalKind::TimedOut,
2335                    workflow_id: description
2336                        .workflow_id
2337                        .clone()
2338                        .unwrap_or_else(|| self.workflow_id.clone()),
2339                    run_id: description.run_id.clone().or_else(|| self.run_id.clone()),
2340                    reason: "result_wait_timeout".to_string(),
2341                    failure_category: Some("client_timeout".to_string()),
2342                    failure_id: None,
2343                    exception_type: None,
2344                    exception_class: None,
2345                    non_retryable: None,
2346                    message: Some(format!(
2347                        "workflow result was not terminal within {:?}",
2348                        options.timeout
2349                    )),
2350                    exception: None,
2351                    raw: description.raw_value(),
2352                }));
2353            }
2354
2355            tokio::time::sleep(options.poll_interval).await;
2356        }
2357    }
2358}
2359
2360#[derive(Clone, Copy, Debug)]
2361pub struct WorkflowResultOptions {
2362    pub poll_interval: Duration,
2363    pub timeout: Duration,
2364}
2365
2366impl Default for WorkflowResultOptions {
2367    fn default() -> Self {
2368        Self {
2369            poll_interval: Duration::from_millis(500),
2370            timeout: Duration::from_secs(30),
2371        }
2372    }
2373}
2374
2375#[derive(Clone, Debug, Deserialize)]
2376pub struct WorkflowDescription {
2377    pub workflow_id: Option<String>,
2378    pub run_id: Option<String>,
2379    pub workflow_type: Option<String>,
2380    pub status: Option<String>,
2381    #[serde(default)]
2382    pub closed_reason: Option<String>,
2383    #[serde(default)]
2384    pub error: Option<String>,
2385    #[serde(default)]
2386    pub failure: Option<Value>,
2387    #[serde(default)]
2388    pub exception: Option<Value>,
2389    #[serde(default)]
2390    pub failures: Vec<Value>,
2391    #[serde(default)]
2392    pub output: Option<Value>,
2393    #[serde(default)]
2394    pub output_envelope: Option<Value>,
2395    #[serde(flatten)]
2396    pub raw: HashMap<String, Value>,
2397}
2398
2399impl WorkflowDescription {
2400    pub fn is_completed(&self) -> bool {
2401        matches!(self.status.as_deref(), Some("completed" | "Completed"))
2402    }
2403
2404    pub fn is_terminal(&self) -> bool {
2405        matches!(
2406            self.status.as_deref(),
2407            Some(
2408                "completed"
2409                    | "Completed"
2410                    | "failed"
2411                    | "Failed"
2412                    | "cancelled"
2413                    | "Cancelled"
2414                    | "terminated"
2415                    | "Terminated"
2416                    | "timed_out"
2417                    | "TimedOut",
2418            )
2419        )
2420    }
2421
2422    fn decode_payloads(&mut self) -> Result<()> {
2423        if let Some(envelope) = &self.output_envelope {
2424            self.output = Some(decode_wire_value(envelope, DEFAULT_CODEC)?);
2425        }
2426
2427        Ok(())
2428    }
2429
2430    fn raw_value(&self) -> Value {
2431        let mut data = self.raw.clone();
2432        data.insert(
2433            "workflow_id".to_string(),
2434            self.workflow_id
2435                .clone()
2436                .map(Value::String)
2437                .unwrap_or(Value::Null),
2438        );
2439        data.insert(
2440            "run_id".to_string(),
2441            self.run_id
2442                .clone()
2443                .map(Value::String)
2444                .unwrap_or(Value::Null),
2445        );
2446        data.insert(
2447            "workflow_type".to_string(),
2448            self.workflow_type
2449                .clone()
2450                .map(Value::String)
2451                .unwrap_or(Value::Null),
2452        );
2453        data.insert(
2454            "status".to_string(),
2455            self.status
2456                .clone()
2457                .map(Value::String)
2458                .unwrap_or(Value::Null),
2459        );
2460        data.insert(
2461            "closed_reason".to_string(),
2462            self.closed_reason
2463                .clone()
2464                .map(Value::String)
2465                .unwrap_or(Value::Null),
2466        );
2467        if let Some(failure) = &self.failure {
2468            data.insert("failure".to_string(), failure.clone());
2469        }
2470        if let Some(exception) = &self.exception {
2471            data.insert("exception".to_string(), exception.clone());
2472        }
2473        Value::Object(data.into_iter().collect())
2474    }
2475}
2476
2477fn workflow_terminal_outcome(
2478    description: &WorkflowDescription,
2479    workflow_id: &str,
2480    run_id: Option<&str>,
2481) -> WorkflowTerminalOutcome {
2482    let terminal_kind = description
2483        .closed_reason
2484        .as_deref()
2485        .or(description.status.as_deref())
2486        .unwrap_or("failed")
2487        .to_ascii_lowercase();
2488    let kind = match terminal_kind.as_str() {
2489        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
2490        "terminated" => WorkflowTerminalKind::Terminated,
2491        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
2492        _ => WorkflowTerminalKind::Failed,
2493    };
2494    let default_reason = match kind {
2495        WorkflowTerminalKind::Failed => "workflow_failed",
2496        WorkflowTerminalKind::Cancelled => "cancelled",
2497        WorkflowTerminalKind::Terminated => "terminated",
2498        WorkflowTerminalKind::TimedOut => "timed_out",
2499    };
2500    let failure = description
2501        .failure
2502        .as_ref()
2503        .filter(|value| value.is_object());
2504    let nested_failure = failure
2505        .and_then(|value| value.get("failures"))
2506        .and_then(Value::as_array)
2507        .and_then(|failures| failures.last())
2508        .or_else(|| description.failures.last());
2509    let exception = description
2510        .exception
2511        .clone()
2512        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
2513        .or_else(|| {
2514            nested_failure
2515                .and_then(|value| value.get("exception_payload"))
2516                .cloned()
2517        });
2518    let string_field = |name: &str| {
2519        failure
2520            .and_then(|value| value.get(name))
2521            .and_then(Value::as_str)
2522            .or_else(|| {
2523                nested_failure
2524                    .and_then(|value| value.get(name))
2525                    .and_then(Value::as_str)
2526            })
2527            .map(str::to_string)
2528    };
2529    let exception_field = |name: &str| {
2530        exception
2531            .as_ref()
2532            .and_then(|value| value.get(name))
2533            .and_then(Value::as_str)
2534            .map(str::to_string)
2535    };
2536    let message = description
2537        .error
2538        .clone()
2539        .or_else(|| string_field("message"))
2540        .or_else(|| exception_field("message"));
2541    let reason = description
2542        .raw
2543        .get("reason")
2544        .and_then(Value::as_str)
2545        .map(str::to_string)
2546        .or_else(|| {
2547            failure
2548                .and_then(|value| value.get("reason"))
2549                .and_then(Value::as_str)
2550                .map(str::to_string)
2551        })
2552        .or_else(|| description.closed_reason.clone())
2553        .unwrap_or_else(|| default_reason.to_string());
2554    let failure_id = string_field("failure_id").or_else(|| {
2555        nested_failure
2556            .and_then(|value| value.get("id"))
2557            .and_then(Value::as_str)
2558            .map(str::to_string)
2559    });
2560
2561    WorkflowTerminalOutcome {
2562        kind,
2563        workflow_id: description
2564            .workflow_id
2565            .clone()
2566            .unwrap_or_else(|| workflow_id.to_string()),
2567        run_id: description
2568            .run_id
2569            .clone()
2570            .or_else(|| run_id.map(str::to_string)),
2571        reason,
2572        failure_category: string_field("failure_category")
2573            .or_else(|| Some(default_reason.to_string())),
2574        failure_id,
2575        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
2576        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
2577        non_retryable: failure
2578            .and_then(|value| value.get("non_retryable"))
2579            .and_then(Value::as_bool)
2580            .or_else(|| {
2581                nested_failure
2582                    .and_then(|value| value.get("non_retryable"))
2583                    .and_then(Value::as_bool)
2584            }),
2585        message,
2586        exception,
2587        raw: description.raw_value(),
2588    }
2589}
2590
2591#[derive(Clone, Debug, Deserialize)]
2592pub struct RegisterWorkerResponse {
2593    pub worker_id: String,
2594    pub registered: bool,
2595    #[serde(default)]
2596    pub heartbeat_interval_seconds: Option<u64>,
2597    #[serde(default)]
2598    pub protocol_version: Option<String>,
2599    #[serde(default)]
2600    pub server_capabilities: Option<Value>,
2601}
2602
2603#[derive(Clone, Debug, Deserialize)]
2604pub struct PollWorkflowTaskResponse {
2605    #[serde(default)]
2606    pub task: Option<WorkflowTask>,
2607    #[serde(default)]
2608    pub poll_status: Option<String>,
2609    #[serde(default)]
2610    pub reason: Option<String>,
2611    #[serde(default)]
2612    pub protocol_version: Option<String>,
2613    #[serde(default)]
2614    pub server_capabilities: Option<Value>,
2615}
2616
2617impl PollWorkflowTaskResponse {
2618    /// Classify this response without parsing server display text.
2619    pub fn outcome(&self) -> WorkerPollOutcome {
2620        worker_poll_outcome(
2621            self.task.is_some(),
2622            self.poll_status.as_deref(),
2623            self.reason.as_deref(),
2624        )
2625    }
2626}
2627
2628#[derive(Clone, Debug, Deserialize)]
2629pub struct PollActivityTaskResponse {
2630    #[serde(default)]
2631    pub task: Option<ActivityTask>,
2632    #[serde(default)]
2633    pub poll_status: Option<String>,
2634    #[serde(default)]
2635    pub reason: Option<String>,
2636}
2637
2638impl PollActivityTaskResponse {
2639    /// Classify this response without parsing server display text.
2640    pub fn outcome(&self) -> WorkerPollOutcome {
2641        worker_poll_outcome(
2642            self.task.is_some(),
2643            self.poll_status.as_deref(),
2644            self.reason.as_deref(),
2645        )
2646    }
2647}
2648
2649#[derive(Clone, Debug, Deserialize)]
2650pub struct PollQueryTaskResponse {
2651    #[serde(default)]
2652    pub task: Option<QueryTask>,
2653    #[serde(default)]
2654    pub poll_status: Option<String>,
2655    #[serde(default)]
2656    pub reason: Option<String>,
2657}
2658
2659impl PollQueryTaskResponse {
2660    /// Classify this response without parsing server display text.
2661    pub fn outcome(&self) -> WorkerPollOutcome {
2662        worker_poll_outcome(
2663            self.task.is_some(),
2664            self.poll_status.as_deref(),
2665            self.reason.as_deref(),
2666        )
2667    }
2668}
2669
2670/// Stable classification for worker poll responses.
2671#[derive(Clone, Debug, PartialEq, Eq)]
2672pub enum WorkerPollOutcome {
2673    /// A task was leased and is available on the response.
2674    Task,
2675    /// No task was leased, but the worker should continue polling.
2676    Idle {
2677        poll_status: Option<String>,
2678        reason: Option<String>,
2679    },
2680    /// The server asked this worker to stop claiming new work.
2681    Stop {
2682        poll_status: Option<String>,
2683        reason: Option<String>,
2684    },
2685}
2686
2687impl WorkerPollOutcome {
2688    pub fn should_stop(&self) -> bool {
2689        matches!(self, Self::Stop { .. })
2690    }
2691}
2692
2693fn worker_poll_outcome(
2694    has_task: bool,
2695    poll_status: Option<&str>,
2696    reason: Option<&str>,
2697) -> WorkerPollOutcome {
2698    if worker_poll_is_stop(poll_status, reason) {
2699        return WorkerPollOutcome::Stop {
2700            poll_status: poll_status.map(str::to_string),
2701            reason: reason.map(str::to_string),
2702        };
2703    }
2704
2705    if has_task {
2706        WorkerPollOutcome::Task
2707    } else {
2708        WorkerPollOutcome::Idle {
2709            poll_status: poll_status.map(str::to_string),
2710            reason: reason.map(str::to_string),
2711        }
2712    }
2713}
2714
2715/// An ephemeral server-routed query task.
2716#[derive(Clone, Debug, Deserialize)]
2717pub struct QueryTask {
2718    pub query_task_id: String,
2719    #[serde(default = "default_workflow_task_attempt")]
2720    pub query_task_attempt: u64,
2721    #[serde(default)]
2722    pub lease_owner: Option<String>,
2723    #[serde(default)]
2724    pub workflow_id: Option<String>,
2725    #[serde(default)]
2726    pub run_id: Option<String>,
2727    pub workflow_type: String,
2728    pub query_name: String,
2729    #[serde(default = "default_payload_codec")]
2730    pub payload_codec: String,
2731    #[serde(default)]
2732    pub workflow_arguments: Option<Value>,
2733    #[serde(default)]
2734    pub query_arguments: Option<Value>,
2735    #[serde(default)]
2736    pub history_events: Vec<HistoryEvent>,
2737    #[serde(default)]
2738    pub history_export: Option<Value>,
2739    #[serde(default)]
2740    pub run_status: Option<String>,
2741}
2742
2743#[derive(Clone, Debug, Deserialize)]
2744pub struct WorkflowTask {
2745    pub task_id: String,
2746    #[serde(default)]
2747    pub workflow_id: Option<String>,
2748    #[serde(default)]
2749    pub run_id: Option<String>,
2750    pub workflow_type: String,
2751    #[serde(default = "default_payload_codec")]
2752    pub payload_codec: String,
2753    #[serde(default)]
2754    pub arguments: Option<Value>,
2755    #[serde(default)]
2756    pub history_events: Vec<HistoryEvent>,
2757    #[serde(default)]
2758    pub total_history_events: Option<u64>,
2759    #[serde(default)]
2760    pub next_history_page_token: Option<String>,
2761    #[serde(default = "default_workflow_task_attempt")]
2762    pub workflow_task_attempt: u64,
2763    #[serde(default)]
2764    pub workflow_signal_id: Option<String>,
2765    #[serde(default)]
2766    pub signal_name: Option<String>,
2767    #[serde(default)]
2768    pub signal_arguments: Option<Value>,
2769    #[serde(default)]
2770    pub lease_owner: Option<String>,
2771}
2772
2773impl WorkflowTask {
2774    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
2775        self.history_events.extend(page.history_events);
2776
2777        if page.total_history_events.is_some() {
2778            self.total_history_events = page.total_history_events;
2779        }
2780
2781        self.next_history_page_token = page
2782            .next_history_page_token
2783            .filter(|token| !token.is_empty());
2784    }
2785}
2786
2787#[derive(Clone, Debug, Deserialize)]
2788struct WorkflowTaskHistoryPage {
2789    #[serde(default)]
2790    history_events: Vec<HistoryEvent>,
2791    #[serde(default)]
2792    total_history_events: Option<u64>,
2793    #[serde(default)]
2794    next_history_page_token: Option<String>,
2795}
2796
2797#[derive(Clone, Debug, Deserialize)]
2798pub struct ActivityTask {
2799    pub task_id: String,
2800    #[serde(default)]
2801    pub activity_attempt_id: Option<String>,
2802    #[serde(default)]
2803    pub attempt_id: Option<String>,
2804    pub activity_type: String,
2805    #[serde(default = "default_payload_codec")]
2806    pub payload_codec: String,
2807    #[serde(default)]
2808    pub arguments: Option<Value>,
2809    #[serde(default = "default_attempt_number")]
2810    pub attempt_number: u64,
2811    #[serde(default)]
2812    pub lease_owner: Option<String>,
2813}
2814
2815#[derive(Clone, Debug, Deserialize)]
2816pub struct HistoryEvent {
2817    #[serde(alias = "type")]
2818    pub event_type: String,
2819    #[serde(default)]
2820    pub payload: Value,
2821    #[serde(flatten)]
2822    pub raw: HashMap<String, Value>,
2823}
2824
2825/// One decoded signal in the committed workflow-history snapshot.
2826#[derive(Clone, Debug, PartialEq)]
2827pub struct QuerySignal {
2828    pub id: Option<String>,
2829    pub name: String,
2830    pub arguments: Vec<Value>,
2831    pub workflow_sequence: Option<u64>,
2832}
2833
2834/// Immutable state supplied to a registered query handler.
2835///
2836/// This context intentionally exposes no activity, signal-wait, or command
2837/// APIs. Query handlers inspect committed history and return a value; query
2838/// completion does not append an event or advance deterministic execution.
2839#[derive(Clone, Debug)]
2840pub struct QueryContext {
2841    pub workflow_id: Option<String>,
2842    pub run_id: Option<String>,
2843    pub workflow_type: String,
2844    pub run_status: Option<String>,
2845    workflow_input: Value,
2846    history_events: Arc<Vec<HistoryEvent>>,
2847    signal_events: Arc<Vec<QuerySignal>>,
2848}
2849
2850impl QueryContext {
2851    /// The normalized argument list used to start the workflow.
2852    pub fn workflow_input(&self) -> &Value {
2853        &self.workflow_input
2854    }
2855
2856    /// The immutable committed history used for this query snapshot.
2857    pub fn history_events(&self) -> &[HistoryEvent] {
2858        self.history_events.as_slice()
2859    }
2860
2861    /// All decoded signals in committed workflow order.
2862    pub fn signal_events(&self) -> &[QuerySignal] {
2863        self.signal_events.as_slice()
2864    }
2865
2866    /// Decoded argument lists for each committed signal with `signal_name`.
2867    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
2868        self.signal_events
2869            .iter()
2870            .filter(|signal| signal.name == signal_name)
2871            .map(|signal| signal.arguments.clone())
2872            .collect()
2873    }
2874}
2875
2876#[derive(Clone, Debug, Deserialize)]
2877pub struct ActivityHeartbeatResponse {
2878    #[serde(default)]
2879    pub cancel_requested: bool,
2880    #[serde(default)]
2881    pub heartbeat_recorded: bool,
2882    #[serde(default)]
2883    pub can_continue: Option<bool>,
2884    #[serde(default)]
2885    pub reason: Option<String>,
2886    #[serde(default)]
2887    pub run_closed_reason: Option<String>,
2888    #[serde(default)]
2889    pub run_closed_at: Option<String>,
2890    #[serde(default)]
2891    pub lease_expires_at: Option<String>,
2892    #[serde(default)]
2893    pub last_heartbeat_at: Option<String>,
2894}
2895
2896impl ActivityHeartbeatResponse {
2897    /// Whether the activity should stop instead of attempting completion.
2898    pub fn should_stop(&self) -> bool {
2899        self.cancel_requested || self.can_continue == Some(false)
2900    }
2901}
2902
2903fn default_payload_codec() -> String {
2904    DEFAULT_CODEC.to_string()
2905}
2906
2907fn default_workflow_task_attempt() -> u64 {
2908    1
2909}
2910
2911fn default_attempt_number() -> u64 {
2912    1
2913}
2914
2915type WorkflowFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2916type WorkflowHandler = Arc<dyn Fn(WorkflowContext, Value) -> WorkflowFuture + Send + Sync>;
2917type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
2918type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
2919type ReplayedWorkflowHandler =
2920    Arc<dyn Fn(WorkflowContext, Value) -> ReplayedWorkflowInvocation + Send + Sync>;
2921type ActivityFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2922type ActivityHandler = Arc<dyn Fn(ActivityContext, Value) -> ActivityFuture + Send + Sync>;
2923type QueryFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2924type QueryHandler = Arc<dyn Fn(QueryContext, Value) -> QueryFuture + Send + Sync>;
2925type ReplayedQueryHandler = Arc<
2926    dyn Fn(QueryContext, ErasedWorkflowState, Value) -> std::result::Result<QueryFuture, String>
2927        + Send
2928        + Sync,
2929>;
2930type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
2931
2932struct ReplayedWorkflowInvocation {
2933    future: WorkflowFuture,
2934    snapshot: WorkflowStateSnapshot,
2935}
2936
2937#[derive(Clone)]
2938struct RegisteredWorkflow {
2939    execute: WorkflowHandler,
2940    replay: Option<ReplayedWorkflowHandler>,
2941    state_type: Option<TypeId>,
2942}
2943
2944#[derive(Clone)]
2945enum RegisteredQuery {
2946    Snapshot(QueryHandler),
2947    Replayed {
2948        state_type: TypeId,
2949        handler: ReplayedQueryHandler,
2950    },
2951}
2952
2953#[derive(Clone, Debug)]
2954pub struct WorkerHeartbeatObservation {
2955    pub worker_id: String,
2956    pub task_queue: String,
2957    pub acknowledged_at_unix_millis: u64,
2958    pub acknowledgement: Value,
2959}
2960
2961/// Bounded retry policy for worker poll acquisition and worker heartbeats.
2962///
2963/// Expected empty long polls are normal successful responses. Transport
2964/// failures, HTTP 408/429 responses, and server errors are retried with capped
2965/// exponential backoff. Authentication, protocol, codec, and handler failures
2966/// are never retried by the worker.
2967#[derive(Clone, Copy, Debug)]
2968pub struct WorkerRetryPolicy {
2969    /// Number of retries after the initial request fails.
2970    pub max_retries: usize,
2971    /// Delay before the first retry.
2972    pub initial_backoff: Duration,
2973    /// Maximum delay between retries.
2974    pub max_backoff: Duration,
2975}
2976
2977impl Default for WorkerRetryPolicy {
2978    fn default() -> Self {
2979        Self {
2980            max_retries: 5,
2981            initial_backoff: Duration::from_millis(100),
2982            max_backoff: Duration::from_secs(5),
2983        }
2984    }
2985}
2986
2987#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2988enum ManagedPollOutcome {
2989    Idle,
2990    Handled,
2991    Stop,
2992}
2993
2994#[derive(Clone)]
2995pub struct Worker {
2996    client: Client,
2997    worker_id: String,
2998    task_queue: String,
2999    workflows: HashMap<String, RegisteredWorkflow>,
3000    activities: HashMap<String, ActivityHandler>,
3001    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
3002    max_concurrent_workflow_tasks: usize,
3003    max_concurrent_activity_tasks: usize,
3004    poll_timeout: Duration,
3005    heartbeat_interval: Duration,
3006    retry_policy: WorkerRetryPolicy,
3007    heartbeat_observer: Option<WorkerHeartbeatObserver>,
3008}
3009
3010impl Worker {
3011    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
3012        Self {
3013            client,
3014            worker_id: default_worker_id(),
3015            task_queue: task_queue.into(),
3016            workflows: HashMap::new(),
3017            activities: HashMap::new(),
3018            queries: HashMap::new(),
3019            max_concurrent_workflow_tasks: 10,
3020            max_concurrent_activity_tasks: 10,
3021            poll_timeout: Duration::from_secs(30),
3022            heartbeat_interval: Duration::from_secs(60),
3023            retry_policy: WorkerRetryPolicy::default(),
3024            heartbeat_observer: None,
3025        }
3026    }
3027
3028    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
3029        self.worker_id = worker_id.into();
3030        self
3031    }
3032
3033    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
3034        self.poll_timeout = timeout;
3035        self
3036    }
3037
3038    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
3039        self.heartbeat_interval = interval;
3040        self
3041    }
3042
3043    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
3044    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
3045        self.retry_policy = policy;
3046        self
3047    }
3048
3049    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
3050    where
3051        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
3052    {
3053        self.heartbeat_observer = Some(Arc::new(observer));
3054        self
3055    }
3056
3057    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
3058        self.max_concurrent_workflow_tasks = count.max(1);
3059        self
3060    }
3061
3062    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
3063        self.max_concurrent_activity_tasks = count.max(1);
3064        self
3065    }
3066
3067    /// Register a workflow handler.
3068    ///
3069    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
3070    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
3071    /// while acquiring or decoding a worker task remain worker-operation
3072    /// failures and do not get converted into workflow outcomes.
3073    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
3074    where
3075        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
3076        Fut: Future<Output = Result<Value>> + Send + 'static,
3077    {
3078        self.workflows.insert(
3079            workflow_type.into(),
3080            RegisteredWorkflow {
3081                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
3082                replay: None,
3083                state_type: None,
3084            },
3085        );
3086    }
3087
3088    /// Register a workflow whose typed instance state can be reconstructed for queries.
3089    ///
3090    /// `state_factory` creates a fresh instance for every normal workflow task and
3091    /// query replay. The workflow handler is the single source of truth for state
3092    /// transitions: it updates [`WorkflowInstance`] after activities and signals
3093    /// resolve. Query replay runs this same handler over committed history and
3094    /// discards any commands it would emit.
3095    pub fn register_replayed_workflow<S, Factory, F, Fut>(
3096        &mut self,
3097        workflow_type: impl Into<String>,
3098        state_factory: Factory,
3099        handler: F,
3100    ) where
3101        S: Clone + Send + Sync + 'static,
3102        Factory: Fn() -> S + Send + Sync + 'static,
3103        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
3104        Fut: Future<Output = Result<Value>> + Send + 'static,
3105    {
3106        let state_factory = Arc::new(state_factory);
3107        let handler = Arc::new(handler);
3108
3109        let execute_factory = Arc::clone(&state_factory);
3110        let execute_handler = Arc::clone(&handler);
3111        let execute = Arc::new(move |ctx: WorkflowContext, input: Value| {
3112            let state = WorkflowInstance::new(execute_factory());
3113            let future = execute_handler(ctx, input, state);
3114            Box::pin(future) as WorkflowFuture
3115        });
3116
3117        let replay = Arc::new(move |ctx: WorkflowContext, input: Value| {
3118            let state = WorkflowInstance::new(state_factory());
3119            let snapshot_state = state.clone();
3120            let snapshot: WorkflowStateSnapshot =
3121                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
3122            let future = handler(ctx, input, state);
3123            ReplayedWorkflowInvocation {
3124                future: Box::pin(future),
3125                snapshot,
3126            }
3127        });
3128
3129        self.workflows.insert(
3130            workflow_type.into(),
3131            RegisteredWorkflow {
3132                execute,
3133                replay: Some(replay),
3134                state_type: Some(TypeId::of::<S>()),
3135            },
3136        );
3137    }
3138
3139    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
3140    where
3141        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
3142        Fut: Future<Output = Result<Value>> + Send + 'static,
3143    {
3144        self.activities.insert(
3145            activity_type.into(),
3146            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
3147        );
3148    }
3149
3150    /// Register a named, read-only query handler for a workflow type.
3151    ///
3152    /// The workflow type must also be registered with [`Worker::register_workflow`]
3153    /// before the worker runs. The handler receives only an immutable committed
3154    /// state snapshot and normalized query arguments.
3155    pub fn register_query<F, Fut>(
3156        &mut self,
3157        workflow_type: impl Into<String>,
3158        query_name: impl Into<String>,
3159        handler: F,
3160    ) where
3161        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
3162        Fut: Future<Output = Result<Value>> + Send + 'static,
3163    {
3164        self.queries
3165            .entry(workflow_type.into())
3166            .or_default()
3167            .insert(
3168                query_name.into(),
3169                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
3170            );
3171    }
3172
3173    /// Register a named query against deterministically replayed instance state.
3174    ///
3175    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
3176    /// same state type `S`. The handler receives an immutable, detached state
3177    /// clone, so successful and failed queries cannot affect workflow execution
3178    /// or the state reconstructed by a later query.
3179    pub fn register_replayed_query<S, F, Fut>(
3180        &mut self,
3181        workflow_type: impl Into<String>,
3182        query_name: impl Into<String>,
3183        handler: F,
3184    ) where
3185        S: Clone + Send + Sync + 'static,
3186        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
3187        Fut: Future<Output = Result<Value>> + Send + 'static,
3188    {
3189        let handler = Arc::new(handler);
3190        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
3191            let state = state.downcast::<S>().map_err(|_| {
3192                "registered query state type does not match the replayed workflow state".to_string()
3193            })?;
3194            Ok(Box::pin(handler(ctx, state, args)))
3195        });
3196
3197        self.queries
3198            .entry(workflow_type.into())
3199            .or_default()
3200            .insert(
3201                query_name.into(),
3202                RegisteredQuery::Replayed {
3203                    state_type: TypeId::of::<S>(),
3204                    handler: erased_handler,
3205                },
3206            );
3207    }
3208
3209    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
3210        self.client
3211            .register_worker_with_capabilities(
3212                &self.worker_id,
3213                &self.task_queue,
3214                self.workflows.keys().cloned().collect(),
3215                self.activities.keys().cloned().collect(),
3216                self.max_concurrent_workflow_tasks,
3217                self.max_concurrent_activity_tasks,
3218                (!self.queries.is_empty())
3219                    .then(|| QUERY_TASKS_CAPABILITY.to_string())
3220                    .into_iter()
3221                    .collect(),
3222            )
3223            .await
3224    }
3225
3226    /// Run until shutdown or a terminal worker error occurs.
3227    ///
3228    /// Empty long-poll expirations do not stop the worker. Retryable poll and
3229    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
3230    /// authentication, protocol, and other non-retryable failures are returned.
3231    pub async fn run(&self) -> Result<()> {
3232        self.run_until(std::future::pending::<()>()).await
3233    }
3234
3235    /// Run until `shutdown` resolves or a terminal worker error occurs.
3236    ///
3237    /// This has the same liveness and terminal-error contract as [`Worker::run`].
3238    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
3239    where
3240        F: Future<Output = ()>,
3241    {
3242        let registration = self.register().await?;
3243        let heartbeat_interval = Duration::from_secs(
3244            registration
3245                .heartbeat_interval_seconds
3246                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
3247        );
3248        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
3249        // from the completion of the preceding attempt, including its bounded
3250        // retries. A fixed-epoch interval can leave an already-due tick queued
3251        // while an acknowledgement is slow, producing a catch-up heartbeat as
3252        // soon as that request completes.
3253        let heartbeat = tokio::time::sleep(Duration::ZERO);
3254        tokio::pin!(heartbeat);
3255        tokio::pin!(shutdown);
3256        let stop = Arc::new(AtomicBool::new(false));
3257        // Poll responses may already have leased server-side work by the time
3258        // they become ready, so each poller owns its responses through
3259        // completion or failure instead of racing raw polls in this select.
3260        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
3261            let worker = self.clone();
3262            let stop = Arc::clone(&stop);
3263            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
3264        });
3265        let mut activity_poller = (!self.activities.is_empty()).then(|| {
3266            let worker = self.clone();
3267            let stop = Arc::clone(&stop);
3268            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
3269        });
3270        let mut query_poller = (!self.queries.is_empty()).then(|| {
3271            let worker = self.clone();
3272            let stop = Arc::clone(&stop);
3273            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
3274        });
3275
3276        loop {
3277            tokio::select! {
3278                _ = &mut shutdown => {
3279                    stop.store(true, Ordering::SeqCst);
3280                    break;
3281                }
3282                _ = &mut heartbeat => {
3283                    let result = self.retry_worker_operation(|| {
3284                        self.client.heartbeat_worker(
3285                            &self.worker_id,
3286                            self.max_concurrent_workflow_tasks,
3287                            self.max_concurrent_activity_tasks,
3288                        )
3289                    }).await;
3290                    heartbeat
3291                        .as_mut()
3292                        .reset(tokio::time::Instant::now() + heartbeat_interval);
3293                    match result {
3294                        Ok(acknowledgement) => {
3295                            if let Some(observer) = &self.heartbeat_observer {
3296                                observer(&WorkerHeartbeatObservation {
3297                                    worker_id: self.worker_id.clone(),
3298                                    task_queue: self.task_queue.clone(),
3299                                    acknowledged_at_unix_millis: SystemTime::now()
3300                                        .duration_since(UNIX_EPOCH)
3301                                        .unwrap_or_default()
3302                                        .as_millis()
3303                                        .min(u64::MAX as u128)
3304                                        as u64,
3305                                    acknowledgement,
3306                                });
3307                            }
3308                        }
3309                        Err(error) => {
3310                            stop.store(true, Ordering::SeqCst);
3311                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
3312                            return Err(error);
3313                        }
3314                    }
3315                }
3316                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
3317                    workflow_poller = None;
3318                    let stopped_by_server = stop.load(Ordering::SeqCst);
3319                    stop.store(true, Ordering::SeqCst);
3320                    let poller_result = optional_poller_result("workflow", result);
3321                    let join_result =
3322                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3323                    poller_result?;
3324                    join_result?;
3325                    if stopped_by_server {
3326                        return Ok(());
3327                    }
3328                    return Err(Error::WorkerLoop(
3329                        "workflow poller stopped unexpectedly".to_string(),
3330                    ));
3331                }
3332                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
3333                    activity_poller = None;
3334                    let stopped_by_server = stop.load(Ordering::SeqCst);
3335                    stop.store(true, Ordering::SeqCst);
3336                    let poller_result = optional_poller_result("activity", result);
3337                    let join_result =
3338                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3339                    poller_result?;
3340                    join_result?;
3341                    if stopped_by_server {
3342                        return Ok(());
3343                    }
3344                    return Err(Error::WorkerLoop(
3345                        "activity poller stopped unexpectedly".to_string(),
3346                    ));
3347                }
3348                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
3349                    query_poller = None;
3350                    let stopped_by_server = stop.load(Ordering::SeqCst);
3351                    stop.store(true, Ordering::SeqCst);
3352                    let poller_result = optional_poller_result("query", result);
3353                    let join_result =
3354                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3355                    poller_result?;
3356                    join_result?;
3357                    if stopped_by_server {
3358                        return Ok(());
3359                    }
3360                    return Err(Error::WorkerLoop(
3361                        "query poller stopped unexpectedly".to_string(),
3362                    ));
3363                }
3364            }
3365        }
3366
3367        join_pollers(
3368            workflow_poller.take(),
3369            activity_poller.take(),
3370            query_poller.take(),
3371        )
3372        .await
3373    }
3374
3375    pub async fn run_once(&self) -> Result<usize> {
3376        let mut handled = 0;
3377        match self.poll_workflow_once().await? {
3378            ManagedPollOutcome::Handled => handled += 1,
3379            ManagedPollOutcome::Stop => return Ok(handled),
3380            ManagedPollOutcome::Idle => {}
3381        }
3382        match self.poll_activity_once().await? {
3383            ManagedPollOutcome::Handled => handled += 1,
3384            ManagedPollOutcome::Stop => return Ok(handled),
3385            ManagedPollOutcome::Idle => {}
3386        }
3387        if !self.queries.is_empty() {
3388            match self.poll_query_once().await? {
3389                ManagedPollOutcome::Handled => handled += 1,
3390                ManagedPollOutcome::Stop => return Ok(handled),
3391                ManagedPollOutcome::Idle => {}
3392            }
3393        }
3394        Ok(handled)
3395    }
3396
3397    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
3398        let response = self
3399            .retry_worker_operation(|| {
3400                self.client.poll_workflow_task_response(
3401                    &self.worker_id,
3402                    &self.task_queue,
3403                    self.poll_timeout,
3404                )
3405            })
3406            .await?;
3407        if response.outcome().should_stop() {
3408            return Ok(ManagedPollOutcome::Stop);
3409        }
3410        let Some(task) = response.task else {
3411            return Ok(ManagedPollOutcome::Idle);
3412        };
3413
3414        let task_id = task.task_id.clone();
3415        let attempt = task.workflow_task_attempt;
3416        let lease_owner = task
3417            .lease_owner
3418            .clone()
3419            .unwrap_or_else(|| self.worker_id.clone());
3420
3421        match self.execute_workflow_task(task) {
3422            Ok(commands) if commands.is_empty() => {
3423                // A replay can consume a recorded pending durable command
3424                // without producing a new command. The standalone protocol
3425                // acknowledges that state through the typed waiting outcome;
3426                // an empty completion is rejected by servers that require at
3427                // least one executable command.
3428                self.client
3429                    .fail_workflow_task_with_type(
3430                        &task_id,
3431                        &lease_owner,
3432                        attempt,
3433                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
3434                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
3435                    )
3436                    .await?;
3437            }
3438            Ok(commands) => {
3439                self.client
3440                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
3441                    .await?;
3442            }
3443            Err(error) => {
3444                self.client
3445                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
3446                    .await?;
3447            }
3448        }
3449
3450        Ok(ManagedPollOutcome::Handled)
3451    }
3452
3453    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3454        while !stop.load(Ordering::SeqCst) {
3455            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
3456                stop.store(true, Ordering::SeqCst);
3457                break;
3458            }
3459        }
3460
3461        Ok(())
3462    }
3463
3464    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
3465        let response = self
3466            .retry_worker_operation(|| {
3467                self.client.poll_activity_task_response(
3468                    &self.worker_id,
3469                    &self.task_queue,
3470                    self.poll_timeout,
3471                )
3472            })
3473            .await?;
3474        if response.outcome().should_stop() {
3475            return Ok(ManagedPollOutcome::Stop);
3476        }
3477        let Some(task) = response.task else {
3478            return Ok(ManagedPollOutcome::Idle);
3479        };
3480
3481        let task_id = task.task_id.clone();
3482        let attempt_id = task
3483            .activity_attempt_id
3484            .clone()
3485            .or(task.attempt_id.clone())
3486            .unwrap_or_default();
3487        let lease_owner = task
3488            .lease_owner
3489            .clone()
3490            .unwrap_or_else(|| self.worker_id.clone());
3491        let codec = task.payload_codec.clone();
3492        let result = self.execute_activity_task(task).await;
3493        match result {
3494            Ok(value) => {
3495                let completion = self
3496                    .client
3497                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
3498                    .await;
3499                if let Err(error) = completion {
3500                    if !activity_task_rejection_is_final(&error) {
3501                        return Err(error);
3502                    }
3503                }
3504            }
3505            Err(error) => {
3506                let failure = self
3507                    .client
3508                    .fail_activity_task(
3509                        &task_id,
3510                        &attempt_id,
3511                        &lease_owner,
3512                        error.to_string(),
3513                        false,
3514                    )
3515                    .await;
3516                if let Err(error) = failure {
3517                    if !activity_task_rejection_is_final(&error) {
3518                        return Err(error);
3519                    }
3520                }
3521            }
3522        }
3523
3524        Ok(ManagedPollOutcome::Handled)
3525    }
3526
3527    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3528        while !stop.load(Ordering::SeqCst) {
3529            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
3530                stop.store(true, Ordering::SeqCst);
3531                break;
3532            }
3533        }
3534
3535        Ok(())
3536    }
3537
3538    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
3539        let response = self
3540            .retry_worker_operation(|| {
3541                self.client.poll_query_task_response(
3542                    &self.worker_id,
3543                    &self.task_queue,
3544                    self.poll_timeout,
3545                )
3546            })
3547            .await?;
3548        if response.outcome().should_stop() {
3549            return Ok(ManagedPollOutcome::Stop);
3550        }
3551        let Some(task) = response.task else {
3552            return Ok(ManagedPollOutcome::Idle);
3553        };
3554
3555        let query_task_id = task.query_task_id.clone();
3556        let attempt = task.query_task_attempt;
3557        let lease_owner = task
3558            .lease_owner
3559            .clone()
3560            .unwrap_or_else(|| self.worker_id.clone());
3561        let codec = task.payload_codec.clone();
3562
3563        match self.execute_query_task(task).await {
3564            Ok(value) => {
3565                let result_envelope = match encode_value_envelope(&value, &codec) {
3566                    Ok(result_envelope) => result_envelope,
3567                    Err(error) => {
3568                        let failure = self
3569                            .client
3570                            .fail_query_task(
3571                                &query_task_id,
3572                                &lease_owner,
3573                                attempt,
3574                                error.to_string(),
3575                                "query_result_encode_failed",
3576                                "QueryResultEncodeFailed",
3577                            )
3578                            .await;
3579                        if let Err(error) = failure {
3580                            if !query_task_rejection_is_final(&error) {
3581                                return Err(error);
3582                            }
3583                        }
3584                        return Ok(ManagedPollOutcome::Handled);
3585                    }
3586                };
3587
3588                if let Err(error) = self
3589                    .client
3590                    .complete_query_task_with_envelope(
3591                        &query_task_id,
3592                        &lease_owner,
3593                        attempt,
3594                        value,
3595                        result_envelope,
3596                    )
3597                    .await
3598                {
3599                    if !query_task_rejection_is_final(&error) {
3600                        return Err(error);
3601                    }
3602                }
3603            }
3604            Err(failure) => {
3605                let result = self
3606                    .client
3607                    .fail_query_task(
3608                        &query_task_id,
3609                        &lease_owner,
3610                        attempt,
3611                        failure.message,
3612                        failure.reason,
3613                        failure.failure_type,
3614                    )
3615                    .await;
3616                if let Err(error) = result {
3617                    if !query_task_rejection_is_final(&error) {
3618                        return Err(error);
3619                    }
3620                }
3621            }
3622        }
3623
3624        Ok(ManagedPollOutcome::Handled)
3625    }
3626
3627    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3628        while !stop.load(Ordering::SeqCst) {
3629            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
3630                stop.store(true, Ordering::SeqCst);
3631                break;
3632            }
3633        }
3634
3635        Ok(())
3636    }
3637
3638    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
3639    where
3640        F: FnMut() -> Fut,
3641        Fut: Future<Output = Result<T>>,
3642    {
3643        let mut retries = 0;
3644
3645        loop {
3646            match operation().await {
3647                Err(error)
3648                    if worker_operation_is_retryable(&error)
3649                        && retries < self.retry_policy.max_retries =>
3650                {
3651                    retries += 1;
3652                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
3653                }
3654                result => return result,
3655            }
3656        }
3657    }
3658
3659    async fn execute_query_task(
3660        &self,
3661        mut task: QueryTask,
3662    ) -> std::result::Result<Value, QueryTaskExecutionFailure> {
3663        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
3664            return Err(QueryTaskExecutionFailure::new(
3665                "query_payload_decode_failed",
3666                format!(
3667                    "cannot decode query payload with unsupported codec {:?}",
3668                    task.payload_codec
3669                ),
3670                "QueryPayloadDecodeFailed",
3671            ));
3672        }
3673
3674        if !self.workflows.contains_key(&task.workflow_type) {
3675            return Err(QueryTaskExecutionFailure::new(
3676                "query_workflow_type_not_registered",
3677                format!("no workflow registered for type {:?}", task.workflow_type),
3678                "WorkflowTypeNotRegistered",
3679            ));
3680        }
3681
3682        let Some(handlers) = self.queries.get(&task.workflow_type) else {
3683            return Err(QueryTaskExecutionFailure::new(
3684                "query_handler_unavailable",
3685                format!(
3686                    "query handlers are unavailable for workflow type {:?}",
3687                    task.workflow_type
3688                ),
3689                "QueryHandlerUnavailable",
3690            ));
3691        };
3692        let Some(query) = handlers.get(&task.query_name) else {
3693            return Err(QueryTaskExecutionFailure::new(
3694                "rejected_unknown_query",
3695                format!("unknown query {:?}", task.query_name),
3696                "QueryFailed",
3697            ));
3698        };
3699
3700        let args = decode_task_arguments(task.query_arguments.as_ref(), &task.payload_codec)
3701            .map_err(|error| {
3702                QueryTaskExecutionFailure::new(
3703                    "query_payload_decode_failed",
3704                    format!("cannot decode query arguments: {error}"),
3705                    "QueryPayloadDecodeFailed",
3706                )
3707            })?;
3708        let workflow_input =
3709            decode_task_arguments(task.workflow_arguments.as_ref(), &task.payload_codec).map_err(
3710                |error| {
3711                    QueryTaskExecutionFailure::new(
3712                        "query_workflow_state_unavailable",
3713                        format!("cannot decode workflow start input: {error}"),
3714                        "QueryWorkflowStateUnavailable",
3715                    )
3716                },
3717            )?;
3718        hydrate_query_history_from_export(&mut task).map_err(|error| {
3719            QueryTaskExecutionFailure::new(
3720                "query_workflow_state_unavailable",
3721                format!("cannot restore query history snapshot: {error}"),
3722                "QueryWorkflowStateUnavailable",
3723            )
3724        })?;
3725        enrich_query_history_from_export(&mut task).map_err(|error| {
3726            QueryTaskExecutionFailure::new(
3727                "query_workflow_state_unavailable",
3728                format!("cannot restore compact query history payloads: {error}"),
3729                "QueryWorkflowStateUnavailable",
3730            )
3731        })?;
3732        let signal_events = query_signal_events(&task).map_err(|error| {
3733            QueryTaskExecutionFailure::new(
3734                "query_workflow_state_unavailable",
3735                format!("cannot decode committed workflow signals: {error}"),
3736                "QueryWorkflowStateUnavailable",
3737            )
3738        })?;
3739        let history_events = Arc::new(std::mem::take(&mut task.history_events));
3740        let context = QueryContext {
3741            workflow_id: task.workflow_id,
3742            run_id: task.run_id,
3743            workflow_type: task.workflow_type.clone(),
3744            run_status: task.run_status,
3745            workflow_input,
3746            history_events: Arc::clone(&history_events),
3747            signal_events: Arc::new(signal_events),
3748        };
3749
3750        let future = match query {
3751            RegisteredQuery::Snapshot(handler) => handler(context, args),
3752            RegisteredQuery::Replayed {
3753                state_type,
3754                handler,
3755            } => {
3756                let workflow = self
3757                    .workflows
3758                    .get(&task.workflow_type)
3759                    .expect("workflow registration was checked above");
3760                if workflow.state_type != Some(*state_type) {
3761                    return Err(QueryTaskExecutionFailure::new(
3762                        "query_workflow_state_unavailable",
3763                        "replayed query state type does not match its workflow registration",
3764                        "QueryWorkflowStateUnavailable",
3765                    ));
3766                }
3767                let replay = workflow.replay.as_ref().ok_or_else(|| {
3768                    QueryTaskExecutionFailure::new(
3769                        "query_workflow_state_unavailable",
3770                        format!(
3771                            "workflow type {:?} is not registered for instance-state replay",
3772                            task.workflow_type
3773                        ),
3774                        "QueryWorkflowStateUnavailable",
3775                    )
3776                })?;
3777                let workflow_state = Arc::new(Mutex::new(
3778                    WorkflowState::new_with_identity(
3779                        history_events.as_ref().clone(),
3780                        context.workflow_id.clone(),
3781                        context.run_id.clone(),
3782                        self.task_queue.clone(),
3783                        task.payload_codec,
3784                        None,
3785                    )
3786                    .map_err(|error| {
3787                        QueryTaskExecutionFailure::new(
3788                            "query_workflow_state_unavailable",
3789                            format!("workflow replay failed before query: {error}"),
3790                            "QueryWorkflowStateUnavailable",
3791                        )
3792                    })?,
3793                ));
3794                let workflow_context = WorkflowContext {
3795                    state: workflow_state,
3796                };
3797                let mut invocation =
3798                    replay(workflow_context.clone(), context.workflow_input.clone());
3799                let mut cx = TaskContext::from_waker(noop_waker_ref());
3800                match invocation.future.as_mut().poll(&mut cx) {
3801                    Poll::Ready(Ok(_)) => {
3802                        workflow_context
3803                            .ensure_history_consumed()
3804                            .map_err(|error| {
3805                                QueryTaskExecutionFailure::new(
3806                                    "query_workflow_state_unavailable",
3807                                    format!("workflow replay failed before query: {error}"),
3808                                    "QueryWorkflowStateUnavailable",
3809                                )
3810                            })?;
3811                    }
3812                    Poll::Ready(Err(error)) => {
3813                        return Err(QueryTaskExecutionFailure::new(
3814                            "query_workflow_state_unavailable",
3815                            format!("workflow replay failed before query: {error}"),
3816                            "QueryWorkflowStateUnavailable",
3817                        ));
3818                    }
3819                    Poll::Pending => {
3820                        let commands = workflow_context.take_commands().map_err(|error| {
3821                            QueryTaskExecutionFailure::new(
3822                                "query_workflow_state_unavailable",
3823                                format!("workflow replay failed before query: {error}"),
3824                                "QueryWorkflowStateUnavailable",
3825                            )
3826                        })?;
3827                        if commands.is_empty()
3828                            && !workflow_context
3829                                .matched_recorded_pending()
3830                                .map_err(|error| {
3831                                    QueryTaskExecutionFailure::new(
3832                                        "query_workflow_state_unavailable",
3833                                        format!("workflow replay failed before query: {error}"),
3834                                        "QueryWorkflowStateUnavailable",
3835                                    )
3836                                })?
3837                        {
3838                            return Err(QueryTaskExecutionFailure::new(
3839                                "query_workflow_state_unavailable",
3840                                "workflow replay yielded without a durable command",
3841                                "QueryWorkflowStateUnavailable",
3842                            ));
3843                        }
3844                    }
3845                }
3846                let state = (invocation.snapshot)().map_err(|error| {
3847                    QueryTaskExecutionFailure::new(
3848                        "query_workflow_state_unavailable",
3849                        format!("cannot snapshot replayed workflow state: {error}"),
3850                        "QueryWorkflowStateUnavailable",
3851                    )
3852                })?;
3853                handler(context, state, args).map_err(|message| {
3854                    QueryTaskExecutionFailure::new(
3855                        "query_workflow_state_unavailable",
3856                        message,
3857                        "QueryWorkflowStateUnavailable",
3858                    )
3859                })?
3860            }
3861        };
3862
3863        future.await.map_err(|error| {
3864            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
3865        })
3866    }
3867
3868    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
3869        let workflow = self
3870            .workflows
3871            .get(&task.workflow_type)
3872            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
3873        let input = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3874        let resume_signal = decode_resume_signal(&task)?;
3875        let state = Arc::new(Mutex::new(WorkflowState::new_with_identity(
3876            task.history_events,
3877            task.workflow_id,
3878            task.run_id,
3879            self.task_queue.clone(),
3880            task.payload_codec.clone(),
3881            resume_signal,
3882        )?));
3883        let ctx = WorkflowContext { state };
3884        let mut future = (workflow.execute)(ctx.clone(), input);
3885        let mut cx = TaskContext::from_waker(noop_waker_ref());
3886
3887        match future.as_mut().poll(&mut cx) {
3888            Poll::Ready(Ok(result)) => {
3889                ctx.ensure_history_consumed()?;
3890                let result = encode_value_envelope(&result, &task.payload_codec)?;
3891                Ok(vec![json!({
3892                    "type": "complete_workflow",
3893                    "result": result
3894                })])
3895            }
3896            Poll::Ready(Err(error)) => Ok(vec![workflow_failure_command(&error)]),
3897            Poll::Pending => {
3898                let commands = ctx.take_commands()?;
3899                if commands.is_empty() && !ctx.matched_recorded_pending()? {
3900                    Err(Error::WorkflowYieldedWithoutCommand)
3901                } else {
3902                    Ok(commands)
3903                }
3904            }
3905        }
3906    }
3907
3908    async fn execute_activity_task(&self, task: ActivityTask) -> Result<Value> {
3909        let handler = self
3910            .activities
3911            .get(&task.activity_type)
3912            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
3913        let args = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3914        let attempt_id = task
3915            .activity_attempt_id
3916            .clone()
3917            .or(task.attempt_id.clone())
3918            .unwrap_or_default();
3919        let lease_owner = task
3920            .lease_owner
3921            .clone()
3922            .unwrap_or_else(|| self.worker_id.clone());
3923        let ctx = ActivityContext {
3924            client: self.client.clone(),
3925            task_id: task.task_id,
3926            activity_attempt_id: attempt_id,
3927            lease_owner,
3928            activity_type: task.activity_type,
3929            attempt_number: task.attempt_number,
3930            task_queue: self.task_queue.clone(),
3931            worker_id: self.worker_id.clone(),
3932        };
3933
3934        handler(ctx, args).await
3935    }
3936}
3937
3938fn poller_result(
3939    kind: &str,
3940    result: std::result::Result<Result<()>, tokio::task::JoinError>,
3941) -> Result<()> {
3942    match result {
3943        Ok(result) => result,
3944        Err(error) => Err(Error::WorkerLoop(format!(
3945            "{kind} poller join error: {error}"
3946        ))),
3947    }
3948}
3949
3950fn optional_poller_result(
3951    kind: &str,
3952    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
3953) -> Result<()> {
3954    match result {
3955        Some(result) => poller_result(kind, result),
3956        None => Ok(()),
3957    }
3958}
3959
3960async fn join_pollers(
3961    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3962    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3963    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3964) -> Result<()> {
3965    let mut first_error = None;
3966
3967    if let Some(handle) = workflow_poller {
3968        if let Err(error) = poller_result("workflow", handle.await) {
3969            first_error.get_or_insert(error);
3970        }
3971    }
3972
3973    if let Some(handle) = activity_poller {
3974        if let Err(error) = poller_result("activity", handle.await) {
3975            first_error.get_or_insert(error);
3976        }
3977    }
3978
3979    if let Some(handle) = query_poller {
3980        if let Err(error) = poller_result("query", handle.await) {
3981            first_error.get_or_insert(error);
3982        }
3983    }
3984
3985    if let Some(error) = first_error {
3986        Err(error)
3987    } else {
3988        Ok(())
3989    }
3990}
3991
3992fn default_worker_id() -> String {
3993    let millis = SystemTime::now()
3994        .duration_since(UNIX_EPOCH)
3995        .unwrap_or_default()
3996        .as_millis();
3997    format!("rust-worker-{}-{millis}", std::process::id())
3998}
3999
4000fn unique_request_id(prefix: &str) -> String {
4001    let nanos = SystemTime::now()
4002        .duration_since(UNIX_EPOCH)
4003        .unwrap_or_default()
4004        .as_nanos();
4005    format!("{prefix}-{}-{nanos}", std::process::id())
4006}
4007
4008#[derive(Debug)]
4009struct QueryTaskExecutionFailure {
4010    reason: String,
4011    message: String,
4012    failure_type: String,
4013}
4014
4015impl QueryTaskExecutionFailure {
4016    fn new(
4017        reason: impl Into<String>,
4018        message: impl Into<String>,
4019        failure_type: impl Into<String>,
4020    ) -> Self {
4021        Self {
4022            reason: reason.into(),
4023            message: message.into(),
4024            failure_type: failure_type.into(),
4025        }
4026    }
4027}
4028
4029/// Typed local state owned by one deterministic workflow invocation.
4030///
4031/// Use [`WorkflowInstance::update`] for the same state transitions during
4032/// ordinary execution and replay. A replayed query receives a detached
4033/// immutable `Arc<S>` rather than this mutation-capable handle.
4034#[derive(Clone, Debug)]
4035pub struct WorkflowInstance<S> {
4036    state: Arc<Mutex<S>>,
4037}
4038
4039impl<S> WorkflowInstance<S> {
4040    fn new(state: S) -> Self {
4041        Self {
4042            state: Arc::new(Mutex::new(state)),
4043        }
4044    }
4045
4046    /// Read the current workflow-instance state without changing it.
4047    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
4048        let state = self
4049            .state
4050            .lock()
4051            .map_err(|_| Error::WorkflowStatePoisoned)?;
4052        Ok(reader(&state))
4053    }
4054
4055    /// Apply one deterministic workflow-instance state transition.
4056    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
4057        let mut state = self
4058            .state
4059            .lock()
4060            .map_err(|_| Error::WorkflowStatePoisoned)?;
4061        Ok(transition(&mut state))
4062    }
4063}
4064
4065impl<S: Clone> WorkflowInstance<S> {
4066    fn snapshot(&self) -> Result<S> {
4067        self.read(Clone::clone)
4068    }
4069}
4070
4071#[derive(Clone, Debug)]
4072pub struct WorkflowContext {
4073    state: Arc<Mutex<WorkflowState>>,
4074}
4075
4076impl WorkflowContext {
4077    /// Identity of the parent workflow currently being replayed.
4078    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
4079        let state = self
4080            .state
4081            .lock()
4082            .map_err(|_| Error::WorkflowStatePoisoned)?;
4083        Ok(WorkflowIdentity {
4084            workflow_id: state.workflow_id.clone(),
4085            run_id: state.run_id.clone(),
4086        })
4087    }
4088
4089    pub fn activity<T: Serialize>(
4090        &self,
4091        activity_type: impl Into<String>,
4092        args: T,
4093    ) -> ActivityCall {
4094        self.activity_with_options(activity_type, ActivityOptions::new(), args)
4095    }
4096
4097    pub fn activity_on_queue<T, Q>(
4098        &self,
4099        activity_type: impl Into<String>,
4100        task_queue: Option<Q>,
4101        args: T,
4102    ) -> ActivityCall
4103    where
4104        T: Serialize,
4105        Q: Into<String>,
4106    {
4107        let mut options = ActivityOptions::new();
4108        options.task_queue = task_queue.map(Into::into);
4109        self.activity_with_options(activity_type, options, args)
4110    }
4111
4112    /// Schedule one durable activity with retry, routing, and timeout options.
4113    ///
4114    /// Options are validated before the command is emitted. Once the command is
4115    /// recorded, replay consumes the same activity lifecycle at this command
4116    /// position and never emits a duplicate schedule.
4117    ///
4118    /// ```no_run
4119    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
4120    /// # use std::time::Duration;
4121    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
4122    /// let result = ctx
4123    ///     .activity_with_options(
4124    ///         "charge-card",
4125    ///         ActivityOptions::new()
4126    ///             .task_queue("payments")
4127    ///             .retry_policy(
4128    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
4129    ///                     Duration::from_secs(1),
4130    ///                     2,
4131    ///                     Some(Duration::from_secs(30)),
4132    ///                 ),
4133    ///             )
4134    ///             .start_to_close_timeout(Duration::from_secs(60))
4135    ///             .schedule_to_close_timeout(Duration::from_secs(180))
4136    ///             .heartbeat_timeout(Duration::from_secs(15)),
4137    ///         json!([{"order_id": "order-42"}]),
4138    ///     )
4139    ///     .await;
4140    /// match result {
4141    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
4142    ///         "reason": failure.reason,
4143    ///         "timeout_kind": failure.timeout_kind,
4144    ///     })),
4145    ///     other => other,
4146    /// }
4147    /// # }
4148    /// ```
4149    pub fn activity_with_options<T: Serialize>(
4150        &self,
4151        activity_type: impl Into<String>,
4152        options: ActivityOptions,
4153        args: T,
4154    ) -> ActivityCall {
4155        ActivityCall {
4156            ctx: self.clone(),
4157            activity_type: activity_type.into(),
4158            options,
4159            args: Some(serde_json::to_value(args).map_err(Error::from)),
4160            scheduled: false,
4161        }
4162    }
4163
4164    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
4165        SignalCall {
4166            ctx: self.clone(),
4167            signal_name: signal_name.into(),
4168            opened_wait: false,
4169            matched_pending: false,
4170        }
4171    }
4172
4173    /// Wait for server-backed durable time without blocking the worker executor.
4174    ///
4175    /// Polling this future emits one `start_timer` command and yields. The
4176    /// server records the deadline, so neither worker nor server restarts reset
4177    /// the wait. Replay resolves the future only from a `TimerScheduled` and
4178    /// `TimerFired` pair at the same position in the shared durable-command
4179    /// stream, with matching sequence, timer identity, and delay. Sub-second
4180    /// durations round up because protocol deadlines use whole seconds.
4181    ///
4182    /// ```no_run
4183    /// # use durable_workflow::{json, Client, Worker};
4184    /// # use std::time::Duration;
4185    /// # fn configure(client: Client) {
4186    /// let mut worker = Worker::new(client, "rust-workers");
4187    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
4188    ///     ctx.sleep(Duration::from_secs(5)).await?;
4189    ///     Ok(json!({"status": "timer fired"}))
4190    /// });
4191    /// # }
4192    /// ```
4193    pub fn sleep(&self, duration: Duration) -> TimerCall {
4194        let delay_seconds = duration
4195            .as_secs()
4196            .checked_add(u64::from(duration.subsec_nanos() > 0));
4197        TimerCall {
4198            ctx: self.clone(),
4199            delay_seconds,
4200            scheduled: false,
4201            matched_pending: false,
4202        }
4203    }
4204
4205    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
4206    pub fn start_timer(&self, duration: Duration) -> TimerCall {
4207        self.sleep(duration)
4208    }
4209
4210    /// Start a named durable child on an explicit queue and await its result.
4211    ///
4212    /// The command is recorded in the parent's sequence-ordered durable command
4213    /// stream. Replay keeps a scheduled child pending without emitting another
4214    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
4215    /// values preserve the history payload codec and include both sides of the
4216    /// durable relationship; failures are returned as
4217    /// [`Error::ChildWorkflowFailed`].
4218    ///
4219    /// ```no_run
4220    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
4221    /// # fn configure(client: Client) {
4222    /// let mut worker = Worker::new(client, "parent-workers");
4223    /// worker.register_workflow("order-parent", |ctx, _input| async move {
4224    ///     let child = ctx
4225    ///         .start_child_workflow(
4226    ///             "fulfil-order",
4227    ///             ChildWorkflowOptions::new("fulfilment-workers")
4228    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
4229    ///             json!([{"order_id": "order-42"}]),
4230    ///         )
4231    ///         .await?;
4232    ///     Ok(child.result)
4233    /// });
4234    /// # }
4235    /// ```
4236    pub fn start_child_workflow<T: Serialize>(
4237        &self,
4238        workflow_type: impl Into<String>,
4239        options: ChildWorkflowOptions,
4240        args: T,
4241    ) -> ChildWorkflowCall {
4242        ChildWorkflowCall {
4243            ctx: self.clone(),
4244            workflow_type: workflow_type.into(),
4245            options,
4246            args: Some(serde_json::to_value(args).map_err(Error::from)),
4247            scheduled: false,
4248            matched_pending: false,
4249        }
4250    }
4251
4252    fn take_commands(&self) -> Result<Vec<Value>> {
4253        let mut state = self
4254            .state
4255            .lock()
4256            .map_err(|_| Error::WorkflowStatePoisoned)?;
4257        Ok(std::mem::take(&mut state.commands))
4258    }
4259
4260    fn matched_recorded_pending(&self) -> Result<bool> {
4261        let state = self
4262            .state
4263            .lock()
4264            .map_err(|_| Error::WorkflowStatePoisoned)?;
4265        Ok(state.matched_recorded_pending)
4266    }
4267
4268    fn ensure_history_consumed(&self) -> Result<()> {
4269        let state = self
4270            .state
4271            .lock()
4272            .map_err(|_| Error::WorkflowStatePoisoned)?;
4273        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
4274            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
4275                "recorded_commands_unconsumed",
4276                Some(command.sequence()),
4277                Some(command.shape().to_string()),
4278                Some("workflow completion".to_string()),
4279                "workflow completed before consuming all recorded durable commands",
4280            )));
4281        }
4282        Ok(())
4283    }
4284}
4285
4286#[derive(Debug)]
4287struct WorkflowState {
4288    history: Vec<HistoryEvent>,
4289    workflow_id: Option<String>,
4290    run_id: Option<String>,
4291    task_queue: String,
4292    payload_codec: String,
4293    resume_signal: Option<ResumeSignal>,
4294    recorded_commands: Vec<RecordedCommand>,
4295    command_cursor: usize,
4296    matched_recorded_pending: bool,
4297    signal_cursors: HashMap<String, usize>,
4298    commands: Vec<Value>,
4299}
4300
4301impl WorkflowState {
4302    #[cfg(test)]
4303    fn new(
4304        history: Vec<HistoryEvent>,
4305        task_queue: String,
4306        payload_codec: String,
4307        resume_signal: Option<ResumeSignal>,
4308    ) -> Result<Self> {
4309        Self::new_with_identity(
4310            history,
4311            None,
4312            None,
4313            task_queue,
4314            payload_codec,
4315            resume_signal,
4316        )
4317    }
4318
4319    fn new_with_identity(
4320        history: Vec<HistoryEvent>,
4321        workflow_id: Option<String>,
4322        run_id: Option<String>,
4323        task_queue: String,
4324        payload_codec: String,
4325        resume_signal: Option<ResumeSignal>,
4326    ) -> Result<Self> {
4327        let recorded_commands = recorded_commands(
4328            &history,
4329            &payload_codec,
4330            WorkflowIdentity {
4331                workflow_id: workflow_id.clone(),
4332                run_id: run_id.clone(),
4333            },
4334        )?;
4335        Ok(Self {
4336            history,
4337            workflow_id,
4338            run_id,
4339            task_queue,
4340            payload_codec,
4341            resume_signal,
4342            recorded_commands,
4343            command_cursor: 0,
4344            matched_recorded_pending: false,
4345            signal_cursors: HashMap::new(),
4346            commands: Vec::new(),
4347        })
4348    }
4349}
4350
4351#[derive(Clone, Debug)]
4352enum RecordedCommand {
4353    Activity {
4354        sequence: u64,
4355        activity_type: Option<String>,
4356        options: Option<RecordedActivityOptions>,
4357        outcome: Option<ActivityOutcome>,
4358    },
4359    Timer {
4360        sequence: u64,
4361        delay_seconds: u64,
4362        fired: bool,
4363    },
4364    ChildWorkflow {
4365        sequence: u64,
4366        workflow_type: Option<String>,
4367        outcome: Option<ChildWorkflowOutcome>,
4368    },
4369    SignalWait {
4370        sequence: u64,
4371        signal_name: Option<String>,
4372    },
4373}
4374
4375#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4376struct RecordedActivityOptions {
4377    task_queue: RecordedSnapshotValue<Option<String>>,
4378    execution_mode: RecordedSnapshotValue<Option<String>>,
4379    retry_policy: ActivityRetrySnapshot,
4380}
4381
4382#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4383enum RecordedSnapshotValue<T> {
4384    /// Older history did not persist this field, so it cannot constrain replay.
4385    Unknown,
4386    Known(T),
4387}
4388
4389impl<T: PartialEq> RecordedSnapshotValue<T> {
4390    fn matches_current(&self, current: &Self) -> bool {
4391        match self {
4392            Self::Unknown => true,
4393            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
4394        }
4395    }
4396}
4397
4398#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4399struct ActivityRetrySnapshot {
4400    snapshot_version: RecordedSnapshotValue<Option<u64>>,
4401    max_attempts: RecordedSnapshotValue<Option<u64>>,
4402    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
4403    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4404    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
4405    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4406    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
4407    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
4408}
4409
4410impl ActivityRetrySnapshot {
4411    fn matches_current(&self, current: &Self) -> bool {
4412        self.snapshot_version
4413            .matches_current(&current.snapshot_version)
4414            && self.max_attempts.matches_current(&current.max_attempts)
4415            && self
4416                .backoff_seconds
4417                .matches_current(&current.backoff_seconds)
4418            && self
4419                .start_to_close_timeout
4420                .matches_current(&current.start_to_close_timeout)
4421            && self
4422                .schedule_to_start_timeout
4423                .matches_current(&current.schedule_to_start_timeout)
4424            && self
4425                .schedule_to_close_timeout
4426                .matches_current(&current.schedule_to_close_timeout)
4427            && self
4428                .heartbeat_timeout
4429                .matches_current(&current.heartbeat_timeout)
4430            && self
4431                .non_retryable_error_types
4432                .matches_current(&current.non_retryable_error_types)
4433    }
4434}
4435
4436fn recorded_optional_u64(
4437    object: Option<&serde_json::Map<String, Value>>,
4438    field: &str,
4439) -> RecordedSnapshotValue<Option<u64>> {
4440    match object.and_then(|object| object.get(field)) {
4441        None => RecordedSnapshotValue::Unknown,
4442        Some(Value::Null) => RecordedSnapshotValue::Known(None),
4443        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
4444    }
4445}
4446
4447fn recorded_optional_string(
4448    object: &serde_json::Map<String, Value>,
4449    field: &str,
4450) -> RecordedSnapshotValue<Option<String>> {
4451    match object.get(field) {
4452        None => RecordedSnapshotValue::Unknown,
4453        Some(Value::Null) => RecordedSnapshotValue::Known(None),
4454        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
4455    }
4456}
4457
4458fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
4459    let policy = policy.and_then(Value::as_object);
4460    let backoff_seconds = policy
4461        .and_then(|policy| policy.get("backoff_seconds"))
4462        .and_then(Value::as_array)
4463        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4464        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
4465    let mut non_retryable_error_types = Vec::new();
4466    for error_type in policy
4467        .and_then(|policy| policy.get("non_retryable_error_types"))
4468        .and_then(Value::as_array)
4469        .into_iter()
4470        .flatten()
4471        .filter_map(Value::as_str)
4472        .map(str::trim)
4473        .filter(|error_type| !error_type.is_empty())
4474    {
4475        if !non_retryable_error_types
4476            .iter()
4477            .any(|recorded| recorded == error_type)
4478        {
4479            non_retryable_error_types.push(error_type.to_string());
4480        }
4481    }
4482
4483    ActivityRetrySnapshot {
4484        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
4485        max_attempts: recorded_optional_u64(policy, "max_attempts"),
4486        backoff_seconds,
4487        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
4488        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
4489        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
4490        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
4491        non_retryable_error_types: if policy
4492            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
4493        {
4494            RecordedSnapshotValue::Known(non_retryable_error_types)
4495        } else {
4496            RecordedSnapshotValue::Unknown
4497        },
4498    }
4499}
4500
4501fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
4502    let policy = options.retry_policy.as_ref();
4503    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
4504        Some(Value::Null) => None,
4505        Some(value) => value_as_u64(value),
4506        None => Some(1),
4507    };
4508    let backoff_seconds = policy
4509        .and_then(|policy| policy.get("backoff_seconds"))
4510        .and_then(Value::as_array)
4511        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4512        .unwrap_or_default();
4513    let non_retryable_error_types = policy
4514        .and_then(|policy| policy.get("non_retryable_error_types"))
4515        .and_then(Value::as_array)
4516        .into_iter()
4517        .flatten()
4518        .filter_map(Value::as_str)
4519        .map(str::to_string)
4520        .collect();
4521
4522    ActivityRetrySnapshot {
4523        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
4524        max_attempts: RecordedSnapshotValue::Known(max_attempts),
4525        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
4526        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
4527        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
4528        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
4529        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
4530        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
4531    }
4532}
4533
4534fn activity_options_description(options: &RecordedActivityOptions) -> String {
4535    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
4536}
4537
4538impl RecordedCommand {
4539    fn sequence(&self) -> u64 {
4540        match self {
4541            Self::Activity { sequence, .. }
4542            | Self::Timer { sequence, .. }
4543            | Self::ChildWorkflow { sequence, .. }
4544            | Self::SignalWait { sequence, .. } => *sequence,
4545        }
4546    }
4547
4548    fn shape(&self) -> &'static str {
4549        match self {
4550            Self::Activity { .. } => "activity",
4551            Self::Timer { .. } => "timer",
4552            Self::ChildWorkflow { .. } => "child workflow",
4553            Self::SignalWait { .. } => "signal wait",
4554        }
4555    }
4556}
4557
4558#[derive(Clone, Debug)]
4559struct ResumeSignal {
4560    signal_id: Option<String>,
4561    signal_name: String,
4562    arguments: Vec<Value>,
4563}
4564
4565pub struct ActivityCall {
4566    ctx: WorkflowContext,
4567    activity_type: String,
4568    options: ActivityOptions,
4569    args: Option<Result<Value>>,
4570    scheduled: bool,
4571}
4572
4573impl Future for ActivityCall {
4574    type Output = Result<Value>;
4575
4576    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4577        let ctx = self.ctx.clone();
4578        let mut state = match ctx.state.lock() {
4579            Ok(state) => state,
4580            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4581        };
4582
4583        if self.scheduled {
4584            return Poll::Pending;
4585        }
4586
4587        let options = match self.options.validate() {
4588            Ok(options) => options,
4589            Err(error) => {
4590                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
4591            }
4592        };
4593        let task_queue = options
4594            .task_queue
4595            .clone()
4596            .unwrap_or_else(|| state.task_queue.clone());
4597        let current_recorded_options = RecordedActivityOptions {
4598            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
4599            // Rust schedules ordinary durable activities. The server records a
4600            // non-null mode only for a specialized execution primitive.
4601            execution_mode: RecordedSnapshotValue::Known(None),
4602            retry_policy: current_activity_retry_snapshot(&options),
4603        };
4604
4605        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4606            let sequence = recorded.sequence();
4607            match recorded {
4608                RecordedCommand::Activity {
4609                    activity_type,
4610                    options: recorded_options,
4611                    outcome,
4612                    ..
4613                } => {
4614                    if let Some(recorded_type) = activity_type {
4615                        if recorded_type != self.activity_type {
4616                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4617                                ReplayFailure::new(
4618                                    "recorded_command_detail_mismatch",
4619                                    Some(sequence),
4620                                    Some(format!("activity:{recorded_type}")),
4621                                    Some(format!("activity:{}", self.activity_type)),
4622                                    "recorded activity type differs from the current workflow command",
4623                                ),
4624                            )));
4625                        }
4626                    }
4627                    if let Some(recorded_options) = recorded_options {
4628                        if !recorded_options
4629                            .task_queue
4630                            .matches_current(&current_recorded_options.task_queue)
4631                        {
4632                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4633                                ReplayFailure::new(
4634                                    "activity_task_queue_mismatch",
4635                                    Some(sequence),
4636                                    Some(activity_options_description(&recorded_options)),
4637                                    Some(activity_options_description(&current_recorded_options)),
4638                                    "recorded activity task queue differs from the current workflow command",
4639                                ),
4640                            )));
4641                        }
4642                        if !recorded_options
4643                            .execution_mode
4644                            .matches_current(&current_recorded_options.execution_mode)
4645                        {
4646                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4647                                ReplayFailure::new(
4648                                    "activity_execution_mode_mismatch",
4649                                    Some(sequence),
4650                                    Some(activity_options_description(&recorded_options)),
4651                                    Some(activity_options_description(&current_recorded_options)),
4652                                    "recorded activity execution mode differs from the current workflow command",
4653                                ),
4654                            )));
4655                        }
4656                        if !recorded_options
4657                            .retry_policy
4658                            .matches_current(&current_recorded_options.retry_policy)
4659                        {
4660                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4661                                ReplayFailure::new(
4662                                    "activity_retry_policy_mismatch",
4663                                    Some(sequence),
4664                                    Some(activity_options_description(&recorded_options)),
4665                                    Some(activity_options_description(&current_recorded_options)),
4666                                    "recorded activity retry policy differs from the current workflow command",
4667                                ),
4668                            )));
4669                        }
4670                    }
4671                    state.command_cursor += 1;
4672                    if let Some(outcome) = outcome {
4673                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
4674                    }
4675                    state.matched_recorded_pending = true;
4676                    self.scheduled = true;
4677                    return Poll::Pending;
4678                }
4679                other => {
4680                    return Poll::Ready(Err(command_mismatch(
4681                        &other,
4682                        format!("activity:{}", self.activity_type),
4683                    )));
4684                }
4685            }
4686        }
4687
4688        if !self.scheduled {
4689            let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4690                Ok(args) => args,
4691                Err(error) => return Poll::Ready(Err(error)),
4692            };
4693            let arguments = normalize_arguments(args);
4694            let envelope = match encode_value_envelope(&arguments, &state.payload_codec) {
4695                Ok(envelope) => envelope,
4696                Err(error) => return Poll::Ready(Err(error)),
4697            };
4698
4699            let mut command = serde_json::Map::from_iter([
4700                ("type".to_string(), json!("schedule_activity")),
4701                (
4702                    "activity_type".to_string(),
4703                    json!(self.activity_type.clone()),
4704                ),
4705                ("queue".to_string(), json!(task_queue)),
4706                ("arguments".to_string(), envelope),
4707            ]);
4708            for (field, value) in [
4709                ("start_to_close_timeout", options.start_to_close_timeout),
4710                (
4711                    "schedule_to_start_timeout",
4712                    options.schedule_to_start_timeout,
4713                ),
4714                (
4715                    "schedule_to_close_timeout",
4716                    options.schedule_to_close_timeout,
4717                ),
4718                ("heartbeat_timeout", options.heartbeat_timeout),
4719            ] {
4720                if let Some(value) = value {
4721                    command.insert(field.to_string(), json!(value));
4722                }
4723            }
4724            if let Some(retry_policy) = options.retry_policy {
4725                command.insert("retry_policy".to_string(), retry_policy);
4726            }
4727            state.commands.push(Value::Object(command));
4728            self.scheduled = true;
4729        }
4730
4731        Poll::Pending
4732    }
4733}
4734
4735/// Future returned by [`WorkflowContext::sleep`].
4736pub struct TimerCall {
4737    ctx: WorkflowContext,
4738    delay_seconds: Option<u64>,
4739    scheduled: bool,
4740    matched_pending: bool,
4741}
4742
4743impl Future for TimerCall {
4744    type Output = Result<()>;
4745
4746    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4747        if self.matched_pending {
4748            return Poll::Pending;
4749        }
4750
4751        let ctx = self.ctx.clone();
4752        let Some(requested_delay) = self.delay_seconds else {
4753            return Poll::Ready(Err(Error::TimerDurationOverflow));
4754        };
4755        let mut state = match ctx.state.lock() {
4756            Ok(state) => state,
4757            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4758        };
4759
4760        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4761            match recorded {
4762                RecordedCommand::Timer {
4763                    sequence,
4764                    delay_seconds,
4765                    fired,
4766                    ..
4767                } => {
4768                    if delay_seconds != requested_delay {
4769                        return Poll::Ready(Err(Error::NonDeterministicReplay(
4770                            ReplayFailure::new(
4771                                "timer_delay_mismatch",
4772                                Some(sequence),
4773                                Some(format!("timer:{delay_seconds}s")),
4774                                Some(format!("timer:{requested_delay}s")),
4775                                "recorded timer delay differs from the current workflow command",
4776                            ),
4777                        )));
4778                    }
4779                    state.command_cursor += 1;
4780                    if fired {
4781                        return Poll::Ready(Ok(()));
4782                    }
4783                    state.matched_recorded_pending = true;
4784                    self.scheduled = true;
4785                    self.matched_pending = true;
4786                    return Poll::Pending;
4787                }
4788                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
4789            }
4790        }
4791
4792        if !self.scheduled {
4793            state.commands.push(json!({
4794                "type": "start_timer",
4795                "delay_seconds": requested_delay,
4796            }));
4797            self.scheduled = true;
4798        }
4799
4800        Poll::Pending
4801    }
4802}
4803
4804/// Future returned by [`WorkflowContext::start_child_workflow`].
4805pub struct ChildWorkflowCall {
4806    ctx: WorkflowContext,
4807    workflow_type: String,
4808    options: ChildWorkflowOptions,
4809    args: Option<Result<Value>>,
4810    scheduled: bool,
4811    matched_pending: bool,
4812}
4813
4814impl Future for ChildWorkflowCall {
4815    type Output = Result<ChildWorkflowResult>;
4816
4817    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4818        if self.matched_pending {
4819            return Poll::Pending;
4820        }
4821
4822        let ctx = self.ctx.clone();
4823        let mut state = match ctx.state.lock() {
4824            Ok(state) => state,
4825            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4826        };
4827
4828        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4829            let sequence = recorded.sequence();
4830            match recorded {
4831                RecordedCommand::ChildWorkflow {
4832                    workflow_type,
4833                    outcome,
4834                    ..
4835                } => {
4836                    if let Some(recorded_type) = workflow_type {
4837                        if recorded_type != self.workflow_type {
4838                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4839                                ReplayFailure::new(
4840                                    "recorded_command_detail_mismatch",
4841                                    Some(sequence),
4842                                    Some(format!("child workflow:{recorded_type}")),
4843                                    Some(format!("child workflow:{}", self.workflow_type)),
4844                                    "recorded child workflow type differs from the current workflow command",
4845                                ),
4846                            )));
4847                        }
4848                    }
4849                    state.command_cursor += 1;
4850                    if let Some(outcome) = outcome {
4851                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
4852                    }
4853                    state.matched_recorded_pending = true;
4854                    self.scheduled = true;
4855                    self.matched_pending = true;
4856                    return Poll::Pending;
4857                }
4858                other => {
4859                    return Poll::Ready(Err(command_mismatch(
4860                        &other,
4861                        format!("child workflow:{}", self.workflow_type),
4862                    )));
4863                }
4864            }
4865        }
4866
4867        if !self.scheduled {
4868            if self.options.task_queue.trim().is_empty() {
4869                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4870                    "task_queue must not be empty".to_string(),
4871                )));
4872            }
4873            for (name, value) in [
4874                (
4875                    "execution_timeout_seconds",
4876                    self.options.execution_timeout_seconds,
4877                ),
4878                ("run_timeout_seconds", self.options.run_timeout_seconds),
4879            ] {
4880                if value == Some(0) {
4881                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
4882                        "{name} must be at least 1"
4883                    ))));
4884                }
4885            }
4886
4887            let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4888                Ok(args) => args,
4889                Err(error) => return Poll::Ready(Err(error)),
4890            };
4891            let arguments =
4892                match encode_value_envelope(&normalize_arguments(args), &state.payload_codec) {
4893                    Ok(arguments) => arguments,
4894                    Err(error) => return Poll::Ready(Err(error)),
4895                };
4896            let mut command = json!({
4897                "type": "start_child_workflow",
4898                "workflow_type": self.workflow_type,
4899                "queue": self.options.task_queue,
4900                "parent_close_policy": self.options.parent_close_policy.as_str(),
4901                "arguments": arguments,
4902            });
4903            let object = command
4904                .as_object_mut()
4905                .expect("child workflow command is always an object");
4906            if let Some(policy) = &self.options.retry_policy {
4907                let mut retry_policy = serde_json::Map::new();
4908                if let Some(max_attempts) = policy.max_attempts {
4909                    if max_attempts == 0 {
4910                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4911                            "retry_policy.max_attempts must be at least 1".to_string(),
4912                        )));
4913                    }
4914                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
4915                }
4916                if !policy.backoff_seconds.is_empty() {
4917                    retry_policy
4918                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
4919                }
4920                if !policy.non_retryable_error_types.is_empty() {
4921                    retry_policy.insert(
4922                        "non_retryable_error_types".to_string(),
4923                        json!(policy.non_retryable_error_types),
4924                    );
4925                }
4926                if retry_policy.is_empty() {
4927                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4928                        "retry_policy must configure at least one field".to_string(),
4929                    )));
4930                }
4931                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
4932            }
4933            if let Some(seconds) = self.options.execution_timeout_seconds {
4934                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
4935            }
4936            if let Some(seconds) = self.options.run_timeout_seconds {
4937                object.insert("run_timeout_seconds".to_string(), json!(seconds));
4938            }
4939            state.commands.push(command);
4940            self.scheduled = true;
4941        }
4942
4943        Poll::Pending
4944    }
4945}
4946
4947fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
4948    Error::NonDeterministicReplay(ReplayFailure::new(
4949        "recorded_command_mismatch",
4950        Some(recorded.sequence()),
4951        Some(recorded.shape().to_string()),
4952        Some(actual.into()),
4953        "current workflow command does not match the recorded durable command sequence",
4954    ))
4955}
4956
4957pub struct SignalCall {
4958    ctx: WorkflowContext,
4959    signal_name: String,
4960    opened_wait: bool,
4961    matched_pending: bool,
4962}
4963
4964impl Future for SignalCall {
4965    type Output = Result<Vec<Value>>;
4966
4967    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4968        if self.matched_pending {
4969            return Poll::Pending;
4970        }
4971
4972        let ctx = self.ctx.clone();
4973        let mut state = match ctx.state.lock() {
4974            Ok(state) => state,
4975            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4976        };
4977
4978        let signals = match signal_values(
4979            &state.history,
4980            &self.signal_name,
4981            &state.payload_codec,
4982            state.resume_signal.as_ref(),
4983        ) {
4984            Ok(signals) => signals,
4985            Err(error) => return Poll::Ready(Err(error)),
4986        };
4987        let cursor = *state.signal_cursors.get(&self.signal_name).unwrap_or(&0);
4988
4989        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4990            match recorded {
4991                RecordedCommand::SignalWait {
4992                    sequence,
4993                    signal_name,
4994                } => {
4995                    if let Some(recorded_name) = signal_name {
4996                        if recorded_name != self.signal_name {
4997                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4998                                ReplayFailure::new(
4999                                    "recorded_command_detail_mismatch",
5000                                    Some(sequence),
5001                                    Some(format!("signal wait:{recorded_name}")),
5002                                    Some(format!("signal wait:{}", self.signal_name)),
5003                                    "recorded signal name differs from the current workflow command",
5004                                ),
5005                            )));
5006                        }
5007                    }
5008
5009                    state.command_cursor += 1;
5010                    if cursor < signals.len() {
5011                        state
5012                            .signal_cursors
5013                            .insert(self.signal_name.clone(), cursor + 1);
5014                        return Poll::Ready(Ok(signals[cursor].clone()));
5015                    }
5016
5017                    state.matched_recorded_pending = true;
5018                    self.opened_wait = true;
5019                    self.matched_pending = true;
5020                    return Poll::Pending;
5021                }
5022                other => {
5023                    return Poll::Ready(Err(command_mismatch(
5024                        &other,
5025                        format!("signal wait:{}", self.signal_name),
5026                    )));
5027                }
5028            }
5029        }
5030
5031        if cursor < signals.len() {
5032            state
5033                .signal_cursors
5034                .insert(self.signal_name.clone(), cursor + 1);
5035            return Poll::Ready(Ok(signals[cursor].clone()));
5036        }
5037
5038        if !self.opened_wait {
5039            state.commands.push(json!({
5040                "type": "open_condition_wait",
5041                "condition_key": format!("signal:{}", self.signal_name)
5042            }));
5043            self.opened_wait = true;
5044        }
5045
5046        Poll::Pending
5047    }
5048}
5049
5050#[derive(Clone, Debug)]
5051pub struct ActivityContext {
5052    client: Client,
5053    pub task_id: String,
5054    pub activity_attempt_id: String,
5055    pub lease_owner: String,
5056    pub activity_type: String,
5057    pub attempt_number: u64,
5058    pub task_queue: String,
5059    pub worker_id: String,
5060}
5061
5062impl ActivityContext {
5063    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
5064        self.client
5065            .heartbeat_activity_task(
5066                &self.task_id,
5067                &self.activity_attempt_id,
5068                &self.lease_owner,
5069                serde_json::to_value(details)?,
5070            )
5071            .await
5072    }
5073}
5074
5075fn decode_task_arguments(value: Option<&Value>, codec: &str) -> Result<Value> {
5076    match value {
5077        Some(value) => Ok(normalize_arguments(decode_wire_value(value, codec)?)),
5078        None => Ok(Value::Array(Vec::new())),
5079    }
5080}
5081
5082fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
5083    let Some(signal_name) = task
5084        .signal_name
5085        .as_deref()
5086        .filter(|value| !value.is_empty())
5087    else {
5088        return Ok(None);
5089    };
5090    let Some(arguments) = task.signal_arguments.as_ref() else {
5091        return Ok(None);
5092    };
5093
5094    let decoded = normalize_arguments(decode_wire_value(arguments, &task.payload_codec)?);
5095    let Value::Array(arguments) = decoded else {
5096        unreachable!("normalize_arguments always returns an array");
5097    };
5098
5099    Ok(Some(ResumeSignal {
5100        signal_id: task.workflow_signal_id.clone(),
5101        signal_name: signal_name.to_string(),
5102        arguments,
5103    }))
5104}
5105
5106fn normalize_arguments(value: Value) -> Value {
5107    match value {
5108        Value::Null => Value::Array(Vec::new()),
5109        Value::Array(_) => value,
5110        other => Value::Array(vec![other]),
5111    }
5112}
5113
5114fn recorded_commands(
5115    events: &[HistoryEvent],
5116    fallback_codec: &str,
5117    parent: WorkflowIdentity,
5118) -> Result<Vec<RecordedCommand>> {
5119    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
5120
5121    for event in events {
5122        let is_activity = matches!(
5123            event.event_type.as_str(),
5124            "ActivityScheduled"
5125                | "ActivityStarted"
5126                | "ActivityHeartbeatRecorded"
5127                | "ActivityRetryScheduled"
5128                | "ActivityCompleted"
5129                | "ActivityFailed"
5130                | "ActivityCancelled"
5131                | "ActivityTimedOut"
5132        );
5133        let is_workflow_timer = matches!(
5134            event.event_type.as_str(),
5135            "TimerScheduled" | "TimerCancelled" | "TimerFired"
5136        ) && !is_internal_timer_event(event);
5137        let is_child_workflow = matches!(
5138            event.event_type.as_str(),
5139            "ChildWorkflowScheduled"
5140                | "ChildRunCompleted"
5141                | "ChildRunFailed"
5142                | "ChildRunCancelled"
5143                | "ChildRunTerminated"
5144        );
5145        let is_signal_wait = is_recorded_signal_wait_event(event);
5146        if !is_activity && !is_workflow_timer && !is_child_workflow && !is_signal_wait {
5147            continue;
5148        }
5149
5150        let sequence = durable_event_sequence(event).ok_or_else(|| {
5151            Error::NonDeterministicReplay(ReplayFailure::new(
5152                "durable_command_sequence_missing",
5153                None,
5154                Some("positive workflow sequence".to_string()),
5155                Some(event.event_type.clone()),
5156                "durable command history event has no workflow sequence",
5157            ))
5158        })?;
5159        if sequence == 0 {
5160            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5161                "durable_command_sequence_invalid",
5162                Some(sequence),
5163                Some("positive workflow sequence".to_string()),
5164                Some(sequence.to_string()),
5165                "durable command history uses an invalid workflow sequence",
5166            )));
5167        }
5168        events_by_sequence.entry(sequence).or_default().push(event);
5169    }
5170
5171    events_by_sequence
5172        .into_iter()
5173        .map(|(sequence, sequence_events)| {
5174            let activity_events: Vec<_> = sequence_events
5175                .iter()
5176                .copied()
5177                .filter(|event| event.event_type.starts_with("Activity"))
5178                .collect();
5179            let timer_events: Vec<_> = sequence_events
5180                .iter()
5181                .copied()
5182                .filter(|event| event.event_type.starts_with("Timer"))
5183                .collect();
5184            let child_events: Vec<_> = sequence_events
5185                .iter()
5186                .copied()
5187                .filter(|event| {
5188                    event.event_type == "ChildWorkflowScheduled"
5189                        || event.event_type.starts_with("ChildRun")
5190                })
5191                .collect();
5192            let signal_wait_events: Vec<_> = sequence_events
5193                .iter()
5194                .copied()
5195                .filter(|event| is_recorded_signal_wait_event(event))
5196                .collect();
5197
5198            let command_kind_count = usize::from(!activity_events.is_empty())
5199                + usize::from(!timer_events.is_empty())
5200                + usize::from(!child_events.is_empty())
5201                + usize::from(!signal_wait_events.is_empty());
5202            if command_kind_count > 1 {
5203                let actual = [
5204                    (!activity_events.is_empty()).then_some("activity"),
5205                    (!timer_events.is_empty()).then_some("timer"),
5206                    (!child_events.is_empty()).then_some("child workflow"),
5207                    (!signal_wait_events.is_empty()).then_some("signal wait"),
5208                ]
5209                .into_iter()
5210                .flatten()
5211                .collect::<Vec<_>>()
5212                .join(" and ");
5213                return Err(invalid_recorded_history(
5214                    "durable_command_sequence_collision",
5215                    sequence,
5216                    "one durable command kind",
5217                    &actual,
5218                    "one workflow sequence records more than one durable command kind",
5219                ));
5220            }
5221
5222            if !activity_events.is_empty() {
5223                let scheduled_count = activity_events
5224                    .iter()
5225                    .filter(|event| event.event_type == "ActivityScheduled")
5226                    .count();
5227                if scheduled_count > 1 {
5228                    return Err(invalid_recorded_history(
5229                        "duplicate_activity_schedule",
5230                        sequence,
5231                        "at most one ActivityScheduled event",
5232                        "multiple ActivityScheduled events",
5233                        "activity history schedules more than one command at one workflow sequence",
5234                    ));
5235                }
5236                let activity_type = activity_events.iter().find_map(|event| {
5237                    event
5238                        .payload
5239                        .get("activity_type")
5240                        .or_else(|| event.payload.get("activity_name"))
5241                        .and_then(Value::as_str)
5242                        .map(str::to_string)
5243                });
5244                if activity_events.iter().filter_map(|event| {
5245                    event
5246                        .payload
5247                        .get("activity_type")
5248                        .or_else(|| event.payload.get("activity_name"))
5249                        .and_then(Value::as_str)
5250                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
5251                    return Err(invalid_recorded_history(
5252                        "activity_identity_mismatch",
5253                        sequence,
5254                        activity_type.as_deref().unwrap_or("one activity identity"),
5255                        "conflicting activity identities",
5256                        "activity lifecycle events at one workflow sequence disagree on identity",
5257                    ));
5258                }
5259                let terminal: Vec<_> = activity_events
5260                    .iter()
5261                    .copied()
5262                    .filter(|event| {
5263                        matches!(
5264                            event.event_type.as_str(),
5265                            "ActivityCompleted"
5266                                | "ActivityFailed"
5267                                | "ActivityCancelled"
5268                                | "ActivityTimedOut"
5269                        )
5270                    })
5271                    .collect();
5272                if terminal.len() > 1 {
5273                    return Err(invalid_recorded_history(
5274                        "duplicate_activity_terminal_event",
5275                        sequence,
5276                        "at most one terminal activity event",
5277                        "multiple terminal activity events",
5278                        "activity history settles one command more than once",
5279                    ));
5280                }
5281                let outcome = terminal
5282                    .first()
5283                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
5284                    .transpose()?;
5285                let options = activity_events
5286                    .iter()
5287                    .find(|event| event.event_type == "ActivityScheduled")
5288                    .and_then(|event| event.payload.get("activity"))
5289                    .and_then(Value::as_object)
5290                    .map(|activity| RecordedActivityOptions {
5291                        task_queue: recorded_optional_string(activity, "queue"),
5292                        execution_mode: recorded_optional_string(activity, "execution_mode"),
5293                        retry_policy: recorded_activity_retry_snapshot(
5294                            activity.get("retry_policy"),
5295                        ),
5296                    });
5297                return Ok(RecordedCommand::Activity {
5298                    sequence,
5299                    activity_type,
5300                    options,
5301                    outcome,
5302                });
5303            }
5304
5305            if !child_events.is_empty() {
5306                let scheduled: Vec<_> = child_events
5307                    .iter()
5308                    .copied()
5309                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
5310                    .collect();
5311                if scheduled.len() != 1 {
5312                    return Err(invalid_recorded_history(
5313                        "child_workflow_schedule_missing_or_duplicate",
5314                        sequence,
5315                        "one ChildWorkflowScheduled event",
5316                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
5317                        "child workflow replay requires exactly one recorded schedule event",
5318                    ));
5319                }
5320                let workflow_type = child_events.iter().find_map(|event| {
5321                    event
5322                        .payload
5323                        .get("child_workflow_type")
5324                        .or_else(|| event.payload.get("workflow_type"))
5325                        .and_then(Value::as_str)
5326                        .filter(|value| !value.is_empty())
5327                        .map(str::to_string)
5328                });
5329                if child_events
5330                    .iter()
5331                    .filter_map(|event| {
5332                        event
5333                            .payload
5334                            .get("child_workflow_type")
5335                            .or_else(|| event.payload.get("workflow_type"))
5336                            .and_then(Value::as_str)
5337                    })
5338                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
5339                {
5340                    return Err(invalid_recorded_history(
5341                        "child_workflow_identity_mismatch",
5342                        sequence,
5343                        workflow_type
5344                            .as_deref()
5345                            .unwrap_or("one child workflow type"),
5346                        "conflicting child workflow types",
5347                        "child workflow lifecycle events at one sequence disagree on type",
5348                    ));
5349                }
5350                let mut outcomes = child_workflow_outcomes(
5351                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
5352                    fallback_codec,
5353                    parent.clone(),
5354                )?;
5355                if outcomes.len() > 1 {
5356                    return Err(invalid_recorded_history(
5357                        "duplicate_child_workflow_terminal_event",
5358                        sequence,
5359                        "at most one terminal child event",
5360                        "multiple terminal child events",
5361                        "child workflow history settles one command more than once",
5362                    ));
5363                }
5364                return Ok(RecordedCommand::ChildWorkflow {
5365                    sequence,
5366                    workflow_type,
5367                    outcome: outcomes.pop(),
5368                });
5369            }
5370
5371            if !signal_wait_events.is_empty() {
5372                let opened_count = signal_wait_events
5373                    .iter()
5374                    .filter(|event| {
5375                        matches!(
5376                            event.event_type.as_str(),
5377                            "SignalWaitOpened" | "ConditionWaitOpened"
5378                        )
5379                    })
5380                    .count();
5381                if opened_count > 1 {
5382                    return Err(invalid_recorded_history(
5383                        "duplicate_signal_wait_open",
5384                        sequence,
5385                        "at most one signal wait open event",
5386                        "multiple signal wait open events",
5387                        "signal history opens more than one durable wait at one workflow sequence",
5388                    ));
5389                }
5390
5391                let signal_name = signal_wait_events
5392                    .iter()
5393                    .find_map(|event| recorded_signal_wait_name(event));
5394                if signal_wait_events
5395                    .iter()
5396                    .filter_map(|event| recorded_signal_wait_name(event))
5397                    .any(|candidate| Some(candidate.as_str()) != signal_name.as_deref())
5398                {
5399                    return Err(invalid_recorded_history(
5400                        "signal_wait_identity_mismatch",
5401                        sequence,
5402                        signal_name.as_deref().unwrap_or("one signal name"),
5403                        "conflicting signal names",
5404                        "signal wait lifecycle events at one workflow sequence disagree on identity",
5405                    ));
5406                }
5407                return Ok(RecordedCommand::SignalWait {
5408                    sequence,
5409                    signal_name,
5410                });
5411            }
5412
5413            let scheduled: Vec<_> = timer_events
5414                .iter()
5415                .copied()
5416                .filter(|event| event.event_type == "TimerScheduled")
5417                .collect();
5418            let fired: Vec<_> = timer_events
5419                .iter()
5420                .copied()
5421                .filter(|event| event.event_type == "TimerFired")
5422                .collect();
5423            if scheduled.len() != 1 {
5424                return Err(invalid_recorded_history(
5425                    "timer_schedule_missing_or_duplicate",
5426                    sequence,
5427                    "one TimerScheduled event",
5428                    &format!("{} TimerScheduled events", scheduled.len()),
5429                    "timer replay requires exactly one recorded schedule event",
5430                ));
5431            }
5432            if fired.len() > 1 {
5433                return Err(invalid_recorded_history(
5434                    "duplicate_timer_fire",
5435                    sequence,
5436                    "at most one TimerFired event",
5437                    "multiple TimerFired events",
5438                    "timer history contains more than one fire event for a workflow sequence",
5439                ));
5440            }
5441
5442            let scheduled = scheduled[0];
5443            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
5444            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
5445            if let Some(fired) = fired.first() {
5446                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
5447                if fired_timer_id != timer_id {
5448                    return Err(invalid_recorded_history(
5449                        "timer_identity_mismatch",
5450                        sequence,
5451                        &timer_id,
5452                        &fired_timer_id,
5453                        "TimerFired does not correspond to the recorded TimerScheduled event",
5454                    ));
5455                }
5456                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
5457                if fired_delay != delay_seconds {
5458                    return Err(invalid_recorded_history(
5459                        "timer_history_delay_mismatch",
5460                        sequence,
5461                        &delay_seconds.to_string(),
5462                        &fired_delay.to_string(),
5463                        "TimerScheduled and TimerFired record different delays",
5464                    ));
5465                }
5466            }
5467
5468            Ok(RecordedCommand::Timer {
5469                sequence,
5470                delay_seconds,
5471                fired: !fired.is_empty(),
5472            })
5473        })
5474        .collect()
5475}
5476
5477fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
5478    event
5479        .payload
5480        .get("sequence")
5481        .or_else(|| event.payload.get("workflow_sequence"))
5482        .or_else(|| event.raw.get("sequence"))
5483        .or_else(|| event.raw.get("workflow_sequence"))
5484        .and_then(value_as_u64)
5485}
5486
5487fn is_internal_timer_event(event: &HistoryEvent) -> bool {
5488    matches!(
5489        event
5490            .payload
5491            .get("timer_kind")
5492            .or_else(|| event.raw.get("timer_kind"))
5493            .and_then(Value::as_str),
5494        Some("condition_timeout" | "signal_timeout")
5495    )
5496}
5497
5498fn recorded_signal_wait_name(event: &HistoryEvent) -> Option<String> {
5499    match event.event_type.as_str() {
5500        "SignalWaitOpened" | "SignalApplied" => event
5501            .payload
5502            .get("signal_name")
5503            .or_else(|| event.raw.get("signal_name"))
5504            .and_then(Value::as_str)
5505            .filter(|value| !value.is_empty())
5506            .map(str::to_string),
5507        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5508            .payload
5509            .get("condition_key")
5510            .or_else(|| event.raw.get("condition_key"))
5511            .and_then(Value::as_str)
5512            .and_then(|key| key.strip_prefix("signal:"))
5513            .filter(|value| !value.is_empty())
5514            .map(str::to_string),
5515        _ => None,
5516    }
5517}
5518
5519fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
5520    match event.event_type.as_str() {
5521        "SignalWaitOpened" | "SignalApplied" => true,
5522        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5523            .payload
5524            .get("condition_key")
5525            .or_else(|| event.raw.get("condition_key"))
5526            .and_then(Value::as_str)
5527            .is_some_and(|key| key.starts_with("signal:")),
5528        _ => false,
5529    }
5530}
5531
5532fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
5533    event
5534        .payload
5535        .get(field)
5536        .and_then(Value::as_str)
5537        .filter(|value| !value.is_empty())
5538        .map(str::to_string)
5539        .ok_or_else(|| {
5540            invalid_recorded_history(
5541                "timer_history_field_missing",
5542                sequence,
5543                field,
5544                &event.event_type,
5545                "timer history is missing a required identity field",
5546            )
5547        })
5548}
5549
5550fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
5551    event
5552        .payload
5553        .get(field)
5554        .and_then(value_as_u64)
5555        .ok_or_else(|| {
5556            invalid_recorded_history(
5557                "timer_history_field_missing",
5558                sequence,
5559                field,
5560                &event.event_type,
5561                "timer history is missing a required numeric field",
5562            )
5563        })
5564}
5565
5566fn invalid_recorded_history(
5567    reason: &str,
5568    sequence: u64,
5569    expected: &str,
5570    actual: &str,
5571    message: &str,
5572) -> Error {
5573    Error::NonDeterministicReplay(ReplayFailure::new(
5574        reason,
5575        Some(sequence),
5576        Some(expected.to_string()),
5577        Some(actual.to_string()),
5578        message,
5579    ))
5580}
5581
5582type ActivityOutcome = std::result::Result<Value, ActivityFailure>;
5583
5584fn activity_outcome(
5585    event: &HistoryEvent,
5586    fallback_codec: &str,
5587    recorded_activity_type: Option<String>,
5588) -> Result<ActivityOutcome> {
5589    if event.event_type == "ActivityCompleted" {
5590        let codec = event
5591            .payload
5592            .get("payload_codec")
5593            .and_then(Value::as_str)
5594            .unwrap_or(fallback_codec);
5595        return Ok(Ok(decode_wire_value(
5596            event.payload.get("result").unwrap_or(&Value::Null),
5597            codec,
5598        )?));
5599    }
5600
5601    let payload = &event.payload;
5602    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
5603        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
5604        "ActivityCancelled" => (
5605            ActivityFailureKind::Cancelled,
5606            "cancelled",
5607            "activity was cancelled",
5608        ),
5609        "ActivityTimedOut" => (
5610            ActivityFailureKind::TimedOut,
5611            "timeout",
5612            "activity timed out",
5613        ),
5614        _ => unreachable!("activity_outcome is called only for terminal activity events"),
5615    };
5616    let exception = payload
5617        .get("exception")
5618        .filter(|value| !value.is_null())
5619        .cloned();
5620    let failure_category = payload_string(payload, "failure_category");
5621    let timeout_kind = payload_string(payload, "timeout_kind");
5622    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
5623        ActivityFailureKind::Failed => failure_category
5624            .clone()
5625            .unwrap_or_else(|| fallback_reason.to_string()),
5626        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
5627        ActivityFailureKind::TimedOut => timeout_kind
5628            .clone()
5629            .unwrap_or_else(|| fallback_reason.to_string()),
5630    });
5631    let message = payload_string(payload, "message")
5632        .or_else(|| {
5633            exception
5634                .as_ref()
5635                .and_then(|value| payload_string(value, "message"))
5636        })
5637        .unwrap_or_else(|| fallback_message.to_string());
5638
5639    Ok(Err(ActivityFailure {
5640        kind,
5641        reason,
5642        message,
5643        activity_execution_id: payload_string(payload, "activity_execution_id"),
5644        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
5645        activity_type: payload_string(payload, "activity_type")
5646            .or_else(|| payload_string(payload, "activity_name"))
5647            .or(recorded_activity_type),
5648        activity_class: payload_string(payload, "activity_class"),
5649        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
5650        failure_id: payload_string(payload, "failure_id"),
5651        failure_category,
5652        timeout_kind,
5653        non_retryable: payload
5654            .get("non_retryable")
5655            .and_then(Value::as_bool)
5656            .unwrap_or(false),
5657        exception_type: payload_string(payload, "exception_type").or_else(|| {
5658            exception
5659                .as_ref()
5660                .and_then(|value| payload_string(value, "type"))
5661        }),
5662        exception_class: payload_string(payload, "exception_class").or_else(|| {
5663            exception
5664                .as_ref()
5665                .and_then(|value| payload_string(value, "class"))
5666        }),
5667        code: payload
5668            .get("code")
5669            .filter(|value| !value.is_null())
5670            .cloned(),
5671        exception,
5672    }))
5673}
5674
5675type ChildWorkflowOutcome = std::result::Result<ChildWorkflowResult, ChildWorkflowFailure>;
5676
5677fn child_workflow_outcomes(
5678    events: &[HistoryEvent],
5679    fallback_codec: &str,
5680    parent: WorkflowIdentity,
5681) -> Result<Vec<ChildWorkflowOutcome>> {
5682    let mut outcomes = Vec::new();
5683
5684    for event in events {
5685        let kind = match event.event_type.as_str() {
5686            "ChildRunCompleted" => None,
5687            "ChildRunFailed" => Some((
5688                ChildWorkflowFailureKind::Failed,
5689                "child_workflow",
5690                "child workflow failed",
5691            )),
5692            "ChildRunCancelled" => Some((
5693                ChildWorkflowFailureKind::Cancelled,
5694                "cancelled",
5695                "child workflow was cancelled",
5696            )),
5697            "ChildRunTerminated" => Some((
5698                ChildWorkflowFailureKind::Terminated,
5699                "terminated",
5700                "child workflow was terminated",
5701            )),
5702            _ => continue,
5703        };
5704        let payload = &event.payload;
5705        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
5706        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
5707        let child_workflow_type = payload_string(payload, "child_workflow_type");
5708
5709        if let Some((kind, reason, fallback_message)) = kind {
5710            let exception = payload
5711                .get("exception")
5712                .filter(|value| !value.is_null())
5713                .cloned();
5714            let message = payload_string(payload, "message")
5715                .or_else(|| {
5716                    exception
5717                        .as_ref()
5718                        .and_then(|value| payload_string(value, "message"))
5719                })
5720                .unwrap_or_else(|| fallback_message.to_string());
5721            let exception_type = payload_string(payload, "exception_type").or_else(|| {
5722                exception
5723                    .as_ref()
5724                    .and_then(|value| payload_string(value, "type"))
5725            });
5726            let exception_class = payload_string(payload, "exception_class").or_else(|| {
5727                exception
5728                    .as_ref()
5729                    .and_then(|value| payload_string(value, "class"))
5730            });
5731            outcomes.push(Err(ChildWorkflowFailure {
5732                kind,
5733                reason: reason.to_string(),
5734                message,
5735                parent_workflow_id: parent.workflow_id.clone(),
5736                parent_workflow_run_id: parent.run_id.clone(),
5737                child_workflow_id,
5738                child_workflow_run_id,
5739                child_workflow_type,
5740                failure_id: payload_string(payload, "failure_id"),
5741                failure_category: payload_string(payload, "failure_category"),
5742                exception_type,
5743                exception_class,
5744                non_retryable: payload
5745                    .get("non_retryable")
5746                    .and_then(Value::as_bool)
5747                    .unwrap_or(false),
5748                code: payload
5749                    .get("code")
5750                    .filter(|value| !value.is_null())
5751                    .cloned(),
5752                exception,
5753            }));
5754            continue;
5755        }
5756
5757        let codec = payload
5758            .get("payload_codec")
5759            .and_then(Value::as_str)
5760            .unwrap_or(fallback_codec);
5761        let result = payload
5762            .get("result")
5763            .or_else(|| payload.get("output"))
5764            .unwrap_or(&Value::Null);
5765        outcomes.push(Ok(ChildWorkflowResult {
5766            parent: parent.clone(),
5767            child: WorkflowIdentity {
5768                workflow_id: child_workflow_id,
5769                run_id: child_workflow_run_id,
5770            },
5771            child_workflow_type,
5772            result: decode_wire_value(result, codec)?,
5773        }));
5774    }
5775
5776    Ok(outcomes)
5777}
5778
5779fn payload_string(payload: &Value, key: &str) -> Option<String> {
5780    payload
5781        .get(key)
5782        .and_then(Value::as_str)
5783        .filter(|value| !value.is_empty())
5784        .map(str::to_string)
5785}
5786
5787fn workflow_failure_command(error: &Error) -> Value {
5788    let (exception_type, exception_class, properties) = match error {
5789        Error::ActivityFailed(failure) => (
5790            match failure.kind {
5791                ActivityFailureKind::Failed => "ActivityFailed",
5792                ActivityFailureKind::Cancelled => "ActivityCancelled",
5793                ActivityFailureKind::TimedOut => "ActivityTimedOut",
5794            },
5795            "durable_workflow::ActivityFailure",
5796            json!({
5797                "reason": failure.reason,
5798                "activity_execution_id": failure.activity_execution_id,
5799                "activity_attempt_id": failure.activity_attempt_id,
5800                "activity_type": failure.activity_type,
5801                "activity_class": failure.activity_class,
5802                "attempt_number": failure.attempt_number,
5803                "failure_id": failure.failure_id,
5804                "failure_category": failure.failure_category,
5805                "timeout_kind": failure.timeout_kind,
5806                "activity_non_retryable": failure.non_retryable,
5807                "activity_exception_type": failure.exception_type,
5808                "activity_exception_class": failure.exception_class,
5809                "activity_code": failure.code,
5810                "activity_exception": failure.exception,
5811            }),
5812        ),
5813        Error::ChildWorkflowFailed(failure) => (
5814            match failure.kind {
5815                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
5816                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
5817                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
5818            },
5819            "durable_workflow::ChildWorkflowFailure",
5820            json!({
5821                "reason": failure.reason,
5822                "parent_workflow_id": failure.parent_workflow_id,
5823                "parent_workflow_run_id": failure.parent_workflow_run_id,
5824                "child_workflow_id": failure.child_workflow_id,
5825                "child_workflow_run_id": failure.child_workflow_run_id,
5826                "child_workflow_type": failure.child_workflow_type,
5827                "failure_id": failure.failure_id,
5828                "failure_category": failure.failure_category,
5829                "child_exception_type": failure.exception_type,
5830                "child_exception_class": failure.exception_class,
5831                "child_non_retryable": failure.non_retryable,
5832                "child_code": failure.code,
5833                "child_exception": failure.exception,
5834            }),
5835        ),
5836        Error::NonDeterministicReplay(_) => (
5837            "NonDeterministicReplay",
5838            "durable_workflow::Error",
5839            Value::Null,
5840        ),
5841        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
5842    };
5843    let non_retryable = match error {
5844        Error::ActivityFailed(failure) => failure.non_retryable,
5845        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
5846        Error::NonDeterministicReplay(_) => true,
5847        _ => false,
5848    };
5849
5850    json!({
5851        "type": "fail_workflow",
5852        "message": error.to_string(),
5853        "exception_type": exception_type,
5854        "exception_class": exception_class,
5855        "non_retryable": non_retryable,
5856        "exception": {
5857            "type": exception_type,
5858            "class": exception_class,
5859            "message": error.to_string(),
5860            "properties": properties,
5861        }
5862    })
5863}
5864
5865fn signal_values(
5866    events: &[HistoryEvent],
5867    signal_name: &str,
5868    fallback_codec: &str,
5869    resume_signal: Option<&ResumeSignal>,
5870) -> Result<Vec<Vec<Value>>> {
5871    let mut signals = Vec::new();
5872
5873    for event in events {
5874        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
5875            continue;
5876        }
5877
5878        if event.payload.get("signal_name").and_then(Value::as_str) != Some(signal_name) {
5879            continue;
5880        }
5881
5882        let codec = event
5883            .payload
5884            .get("payload_codec")
5885            .and_then(Value::as_str)
5886            .unwrap_or(fallback_codec);
5887        let raw = event
5888            .payload
5889            .get("value")
5890            .or_else(|| event.payload.get("input"))
5891            .or_else(|| event.payload.get("arguments"));
5892        let decoded = match raw.filter(|value| !value.is_null()) {
5893            Some(value) => decode_wire_value(value, codec)?,
5894            None => resume_signal
5895                .filter(|signal| resume_signal_matches_event(signal, event, signal_name))
5896                .map(|signal| Value::Array(signal.arguments.clone()))
5897                .unwrap_or_else(|| Value::Array(Vec::new())),
5898        };
5899        let args = match normalize_arguments(decoded) {
5900            Value::Array(values) => values,
5901            _ => unreachable!("normalize_arguments always returns an array"),
5902        };
5903        signals.push(args);
5904    }
5905
5906    Ok(signals)
5907}
5908
5909fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5910    let Some(export_events) = task
5911        .history_export
5912        .as_ref()
5913        .and_then(|export| export.get("history_events"))
5914        .and_then(Value::as_array)
5915    else {
5916        return Ok(());
5917    };
5918
5919    if export_events.len() > task.history_events.len() {
5920        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
5921    }
5922
5923    Ok(())
5924}
5925
5926fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5927    let Some(export) = task.history_export.as_ref() else {
5928        return Ok(());
5929    };
5930    let signals = export
5931        .get("signals")
5932        .and_then(Value::as_array)
5933        .cloned()
5934        .unwrap_or_default();
5935    let activities = export
5936        .get("activities")
5937        .and_then(Value::as_array)
5938        .cloned()
5939        .unwrap_or_default();
5940    let export_codec = export
5941        .get("payloads")
5942        .and_then(|payloads| payloads.get("codec"))
5943        .and_then(Value::as_str)
5944        .unwrap_or(&task.payload_codec)
5945        .to_string();
5946    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
5947
5948    for event in &mut task.history_events {
5949        if event.event_type == "ActivityCompleted" {
5950            let sequence = event
5951                .payload
5952                .get("sequence")
5953                .or_else(|| event.payload.get("workflow_sequence"))
5954                .and_then(value_as_u64);
5955            let Some(activity) = sequence.and_then(|sequence| {
5956                activities.iter().find(|activity| {
5957                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
5958                })
5959            }) else {
5960                continue;
5961            };
5962            let Some(payload) = event.payload.as_object_mut() else {
5963                continue;
5964            };
5965            if missing_payload(payload.get("result")) {
5966                if let Some(result) = activity
5967                    .get("result")
5968                    .filter(|value| !missing_payload(Some(value)))
5969                {
5970                    payload.insert("result".to_string(), result.clone());
5971                }
5972            }
5973            for field in ["payload_codec", "activity_type"] {
5974                if payload
5975                    .get(field)
5976                    .and_then(Value::as_str)
5977                    .unwrap_or_default()
5978                    .is_empty()
5979                {
5980                    if let Some(value) = activity.get(field) {
5981                        payload.insert(field.to_string(), value.clone());
5982                    }
5983                }
5984            }
5985            continue;
5986        }
5987
5988        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
5989            continue;
5990        }
5991        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
5992        let command_id = event
5993            .payload
5994            .get("workflow_command_id")
5995            .or_else(|| event.raw.get("workflow_command_id"))
5996            .and_then(Value::as_str);
5997        let signal_name = event
5998            .payload
5999            .get("signal_name")
6000            .and_then(Value::as_str)
6001            .unwrap_or_default()
6002            .to_string();
6003        let matched = signals
6004            .iter()
6005            .find(|signal| {
6006                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
6007            })
6008            .or_else(|| {
6009                signals.iter().find(|signal| {
6010                    command_id.is_some()
6011                        && signal.get("command_id").and_then(Value::as_str) == command_id
6012                })
6013            })
6014            .or_else(|| {
6015                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
6016                let signal = signals
6017                    .iter()
6018                    .filter(|signal| {
6019                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
6020                    })
6021                    .nth(*offset);
6022                if signal.is_some() {
6023                    *offset += 1;
6024                }
6025                signal
6026            });
6027        let Some(signal) = matched else {
6028            continue;
6029        };
6030        let signal_codec = signal
6031            .get("payload_codec")
6032            .and_then(Value::as_str)
6033            .unwrap_or(&export_codec);
6034        let Some(payload) = event.payload.as_object_mut() else {
6035            continue;
6036        };
6037        if missing_payload(payload.get("arguments")) {
6038            if let Some(arguments) = signal
6039                .get("arguments")
6040                .filter(|value| !missing_payload(Some(value)))
6041            {
6042                let envelope = match arguments {
6043                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
6044                    other => other.clone(),
6045                };
6046                payload.insert("arguments".to_string(), envelope);
6047            }
6048        }
6049        if payload
6050            .get("payload_codec")
6051            .and_then(Value::as_str)
6052            .unwrap_or_default()
6053            .is_empty()
6054        {
6055            payload.insert("payload_codec".to_string(), json!(signal_codec));
6056        }
6057    }
6058
6059    Ok(())
6060}
6061
6062fn missing_payload(value: Option<&Value>) -> bool {
6063    match value {
6064        None | Some(Value::Null) => true,
6065        Some(Value::String(value)) => value.is_empty(),
6066        Some(_) => false,
6067    }
6068}
6069
6070fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
6071    let export_signals = task
6072        .history_export
6073        .as_ref()
6074        .and_then(|export| export.get("signals"))
6075        .and_then(Value::as_array)
6076        .cloned()
6077        .unwrap_or_default();
6078    let export_codec = task
6079        .history_export
6080        .as_ref()
6081        .and_then(|export| export.get("payloads"))
6082        .and_then(|payloads| payloads.get("codec"))
6083        .and_then(Value::as_str)
6084        .unwrap_or(&task.payload_codec);
6085    let mut name_offsets: HashMap<String, usize> = HashMap::new();
6086    let mut signals = Vec::new();
6087
6088    for event in &task.history_events {
6089        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
6090            continue;
6091        }
6092
6093        let name = event
6094            .payload
6095            .get("signal_name")
6096            .and_then(Value::as_str)
6097            .unwrap_or_default();
6098        if name.is_empty() {
6099            continue;
6100        }
6101        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
6102        let command_id = event
6103            .payload
6104            .get("workflow_command_id")
6105            .or_else(|| event.raw.get("workflow_command_id"))
6106            .and_then(Value::as_str);
6107        let matched_export = export_signals
6108            .iter()
6109            .find(|candidate| {
6110                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
6111            })
6112            .or_else(|| {
6113                export_signals.iter().find(|candidate| {
6114                    command_id.is_some()
6115                        && candidate.get("command_id").and_then(Value::as_str) == command_id
6116                })
6117            })
6118            .or_else(|| {
6119                let offset = name_offsets.entry(name.to_string()).or_default();
6120                let candidate = export_signals
6121                    .iter()
6122                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
6123                    .nth(*offset);
6124                if candidate.is_some() {
6125                    *offset += 1;
6126                }
6127                candidate
6128            });
6129        let codec = event
6130            .payload
6131            .get("payload_codec")
6132            .and_then(Value::as_str)
6133            .or_else(|| {
6134                matched_export
6135                    .and_then(|signal| signal.get("payload_codec"))
6136                    .and_then(Value::as_str)
6137            })
6138            .unwrap_or(export_codec);
6139        let raw_arguments = event
6140            .payload
6141            .get("value")
6142            .or_else(|| event.payload.get("input"))
6143            .or_else(|| event.payload.get("arguments"))
6144            .filter(|value| !value.is_null())
6145            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
6146        let arguments = decode_query_signal_arguments(raw_arguments, codec)?;
6147        let workflow_sequence = event
6148            .payload
6149            .get("workflow_sequence")
6150            .and_then(value_as_u64)
6151            .or_else(|| {
6152                matched_export
6153                    .and_then(|signal| signal.get("workflow_sequence"))
6154                    .and_then(value_as_u64)
6155            });
6156
6157        signals.push(QuerySignal {
6158            id: signal_id.map(str::to_string).or_else(|| {
6159                matched_export
6160                    .and_then(|signal| signal.get("id"))
6161                    .and_then(Value::as_str)
6162                    .map(str::to_string)
6163            }),
6164            name: name.to_string(),
6165            arguments,
6166            workflow_sequence,
6167        });
6168    }
6169
6170    if signals.is_empty() {
6171        for signal in export_signals {
6172            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
6173                continue;
6174            }
6175            let Some(name) = signal.get("name").and_then(Value::as_str) else {
6176                continue;
6177            };
6178            let codec = signal
6179                .get("payload_codec")
6180                .and_then(Value::as_str)
6181                .unwrap_or(export_codec);
6182            let arguments = decode_query_signal_arguments(signal.get("arguments"), codec)?;
6183            signals.push(QuerySignal {
6184                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
6185                name: name.to_string(),
6186                arguments,
6187                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
6188            });
6189        }
6190        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
6191    }
6192
6193    Ok(signals)
6194}
6195
6196fn decode_query_signal_arguments(raw: Option<&Value>, codec: &str) -> Result<Vec<Value>> {
6197    let decoded = match raw.filter(|value| !value.is_null()) {
6198        Some(value) => decode_wire_value(value, codec)?,
6199        None => Value::Array(Vec::new()),
6200    };
6201    let Value::Array(arguments) = normalize_arguments(decoded) else {
6202        unreachable!("normalize_arguments always returns an array");
6203    };
6204    Ok(arguments)
6205}
6206
6207fn value_as_u64(value: &Value) -> Option<u64> {
6208    value
6209        .as_u64()
6210        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
6211}
6212
6213fn resume_signal_matches_event(
6214    resume_signal: &ResumeSignal,
6215    event: &HistoryEvent,
6216    signal_name: &str,
6217) -> bool {
6218    if resume_signal.signal_name != signal_name {
6219        return false;
6220    }
6221
6222    match (
6223        resume_signal.signal_id.as_deref(),
6224        event.payload.get("signal_id").and_then(Value::as_str),
6225    ) {
6226        (Some(resume_id), Some(event_id)) => resume_id == event_id,
6227        _ => true,
6228    }
6229}
6230
6231#[cfg(test)]
6232mod tests {
6233    use super::*;
6234    use std::{
6235        io::{Read, Write},
6236        net::{SocketAddr, TcpListener, TcpStream},
6237        thread,
6238    };
6239
6240    #[derive(Clone, Debug, Default, PartialEq)]
6241    struct ReplayCounterState {
6242        loaded: Option<String>,
6243        count: i64,
6244        finished: bool,
6245    }
6246
6247    fn replay_counter_worker() -> Worker {
6248        let client = Client::new("http://127.0.0.1:8080").expect("client");
6249        let mut worker = Worker::new(client, "rust-workers");
6250        worker.register_replayed_workflow(
6251            "replay-counter",
6252            ReplayCounterState::default,
6253            |ctx, _input, state| async move {
6254                let loaded = ctx.activity("load-counter", json!([])).await?;
6255                state.update(|current| {
6256                    current.loaded = loaded.as_str().map(str::to_string);
6257                })?;
6258                for _ in 0..2 {
6259                    let signal = ctx.wait_signal("increment").await?;
6260                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
6261                    state.update(|current| current.count += amount)?;
6262                }
6263                state.update(|current| current.finished = true)?;
6264                state.read(|current| Ok(json!(current.count)))?
6265            },
6266        );
6267        worker.register_replayed_query::<ReplayCounterState, _, _>(
6268            "replay-counter",
6269            "current",
6270            |_ctx, state, _args| async move {
6271                Ok(json!({
6272                    "loaded": state.loaded,
6273                    "count": state.count,
6274                    "finished": state.finished,
6275                }))
6276            },
6277        );
6278        worker.register_replayed_query::<ReplayCounterState, _, _>(
6279            "replay-counter",
6280            "detached-mutation",
6281            |_ctx, state, _args| async move {
6282                let mut detached = (*state).clone();
6283                detached.count = 999;
6284                Ok(json!(detached.count))
6285            },
6286        );
6287        worker.register_replayed_query::<ReplayCounterState, _, _>(
6288            "replay-counter",
6289            "failed-mutation",
6290            |_ctx, state, _args| async move {
6291                let mut detached = (*state).clone();
6292                detached.count = 999;
6293                Err(Error::WorkerLoop("query refused".to_string()))
6294            },
6295        );
6296        worker
6297    }
6298
6299    fn replay_counter_query(
6300        query_name: &str,
6301        history_events: Value,
6302        run_status: &str,
6303    ) -> QueryTask {
6304        serde_json::from_value(json!({
6305            "query_task_id": format!("query-{query_name}"),
6306            "workflow_type": "replay-counter",
6307            "query_name": query_name,
6308            "payload_codec": "json",
6309            "workflow_arguments": {"codec": "json", "blob": "[]"},
6310            "query_arguments": {"codec": "json", "blob": "[]"},
6311            "history_events": history_events,
6312            "run_status": run_status,
6313        }))
6314        .expect("query task")
6315    }
6316
6317    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
6318        WorkflowContext {
6319            state: Arc::new(Mutex::new(
6320                WorkflowState::new_with_identity(
6321                    history,
6322                    None,
6323                    None,
6324                    "rust-workers".to_string(),
6325                    JSON_CODEC.to_string(),
6326                    None,
6327                )
6328                .expect("valid workflow history"),
6329            )),
6330        }
6331    }
6332
6333    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
6334        HistoryEvent {
6335            event_type: event_type.to_string(),
6336            payload,
6337            raw: HashMap::new(),
6338        }
6339    }
6340
6341    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
6342        vec![
6343            history_event(
6344                "ActivityScheduled",
6345                json!({
6346                    "sequence": 1,
6347                    "activity_type": "flaky",
6348                    "activity_execution_id": "act-1",
6349                    "activity": {
6350                        "id": "act-1",
6351                        "sequence": 1,
6352                        "type": "flaky",
6353                        "queue": "critical-activities",
6354                        "execution_mode": null,
6355                        "retry_policy": {
6356                            "snapshot_version": 1,
6357                            "max_attempts": 3,
6358                            "backoff_seconds": [2, 4],
6359                            "start_to_close_timeout": 30,
6360                            "schedule_to_start_timeout": 5,
6361                            "schedule_to_close_timeout": 90,
6362                            "heartbeat_timeout": 10,
6363                            "non_retryable_error_types": ["PermanentError"]
6364                        }
6365                    }
6366                }),
6367            ),
6368            history_event(
6369                "ActivityStarted",
6370                json!({
6371                    "sequence": 1,
6372                    "activity_type": "flaky",
6373                    "activity_execution_id": "act-1",
6374                    "activity_attempt_id": "attempt-1",
6375                    "attempt_number": 1
6376                }),
6377            ),
6378            history_event(
6379                "ActivityRetryScheduled",
6380                json!({
6381                    "sequence": 1,
6382                    "activity_type": "flaky",
6383                    "activity_execution_id": "act-1",
6384                    "activity_attempt_id": "attempt-1",
6385                    "attempt_number": 1,
6386                    "retry_after_attempt": 1,
6387                    "retry_backoff_seconds": 2,
6388                    "failure_category": "activity",
6389                    "exception_type": "TransientError"
6390                }),
6391            ),
6392            history_event(
6393                "ActivityStarted",
6394                json!({
6395                    "sequence": 1,
6396                    "activity_type": "flaky",
6397                    "activity_execution_id": "act-1",
6398                    "activity_attempt_id": "attempt-2",
6399                    "attempt_number": 2
6400                }),
6401            ),
6402            history_event(
6403                "ActivityCompleted",
6404                json!({
6405                    "sequence": 1,
6406                    "activity_type": "flaky",
6407                    "activity_execution_id": "act-1",
6408                    "activity_attempt_id": "attempt-2",
6409                    "attempt_number": 2,
6410                    "payload_codec": "json",
6411                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
6412                }),
6413            ),
6414        ]
6415    }
6416
6417    fn retry_activity_options() -> ActivityOptions {
6418        ActivityOptions::new()
6419            .task_queue("critical-activities")
6420            .retry_policy(
6421                ActivityRetryPolicy::new(3)
6422                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
6423                    .non_retryable_error_type("PermanentError"),
6424            )
6425            .start_to_close_timeout(Duration::from_secs(30))
6426            .schedule_to_start_timeout(Duration::from_secs(5))
6427            .schedule_to_close_timeout(Duration::from_secs(90))
6428            .heartbeat_timeout(Duration::from_secs(10))
6429    }
6430
6431    #[test]
6432    fn avro_generic_wrapper_round_trips_json_values() {
6433        let value = json!({"greeting": "hello", "count": 3, "ok": true});
6434        let envelope = PayloadEnvelope::avro(&value).expect("encode");
6435        assert_eq!(envelope.codec, DEFAULT_CODEC);
6436        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6437    }
6438
6439    #[test]
6440    fn json_codec_remains_plain_json() {
6441        let value = json!({"greeting": "hello", "count": 3, "ok": true});
6442        let envelope = PayloadEnvelope::json(&value).expect("encode");
6443
6444        assert_eq!(envelope.codec, JSON_CODEC);
6445        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
6446        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6447    }
6448
6449    #[test]
6450    fn typed_avro_payload_without_schema_context_keeps_diagnostic() {
6451        let envelope = PayloadEnvelope {
6452            codec: DEFAULT_CODEC.to_string(),
6453            blob: BASE64.encode([0x01]),
6454        };
6455
6456        let error = decode_payload::<Value>(&envelope).expect_err("typed payload must fail");
6457        assert_eq!(
6458            error.to_string(),
6459            "codec error: typed avro payloads require a schema context; v1 supports the generic wrapper"
6460        );
6461    }
6462
6463    #[test]
6464    fn workflow_context_schedules_activity_until_completion_is_in_history() {
6465        let ctx = WorkflowContext {
6466            state: Arc::new(Mutex::new(
6467                WorkflowState::new_with_identity(
6468                    Vec::new(),
6469                    Some("wf-parent".to_string()),
6470                    Some("run-parent".to_string()),
6471                    "rust-workers".to_string(),
6472                    DEFAULT_CODEC.to_string(),
6473                    None,
6474                )
6475                .expect("workflow state"),
6476            )),
6477        };
6478
6479        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
6480        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6481        assert!(matches!(
6482            call.as_mut().poll(&mut task_context),
6483            Poll::Pending
6484        ));
6485
6486        let commands = ctx.take_commands().expect("commands");
6487        assert_eq!(commands[0]["type"], "schedule_activity");
6488        assert_eq!(commands[0]["activity_type"], "hello.activity");
6489    }
6490
6491    #[test]
6492    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
6493        let ctx = workflow_context(Vec::new());
6494        let options = ActivityOptions::new()
6495            .task_queue("payments")
6496            .retry_policy(
6497                ActivityRetryPolicy::new(4)
6498                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
6499                    .non_retryable_error_type("ValidationError"),
6500            )
6501            .start_to_close_timeout(Duration::from_secs(120))
6502            .schedule_to_start_timeout(Duration::from_secs(10))
6503            .schedule_to_close_timeout(Duration::from_secs(300))
6504            .heartbeat_timeout(Duration::from_secs(15));
6505        let mut call = Box::pin(ctx.activity_with_options(
6506            "charge-card",
6507            options,
6508            json!([{"order_id": "o-1"}]),
6509        ));
6510        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6511
6512        assert!(matches!(
6513            call.as_mut().poll(&mut task_context),
6514            Poll::Pending
6515        ));
6516        assert!(matches!(
6517            call.as_mut().poll(&mut task_context),
6518            Poll::Pending
6519        ));
6520
6521        let commands = ctx.take_commands().expect("activity command");
6522        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
6523        assert_eq!(commands[0]["queue"], "payments");
6524        assert_eq!(
6525            commands[0]["retry_policy"],
6526            json!({
6527                "max_attempts": 4,
6528                "backoff_seconds": [1, 3, 9],
6529                "non_retryable_error_types": ["ValidationError"],
6530            })
6531        );
6532        assert_eq!(commands[0]["start_to_close_timeout"], 120);
6533        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
6534        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
6535        assert_eq!(commands[0]["heartbeat_timeout"], 15);
6536    }
6537
6538    #[test]
6539    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
6540        let ctx = workflow_context(Vec::new());
6541        let options = ActivityOptions::new().retry_policy(
6542            ActivityRetryPolicy::new(3)
6543                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
6544        );
6545        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6546        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6547
6548        assert!(matches!(
6549            call.as_mut().poll(&mut task_context),
6550            Poll::Pending
6551        ));
6552        assert_eq!(
6553            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
6554            json!([1, 2])
6555        );
6556    }
6557
6558    #[test]
6559    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
6560        let cases = [
6561            (
6562                ActivityOptions::new().task_queue("  "),
6563                ActivityOptionsErrorKind::EmptyTaskQueue,
6564            ),
6565            (
6566                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
6567                ActivityOptionsErrorKind::EmptyRetryPolicy,
6568            ),
6569            (
6570                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
6571                ActivityOptionsErrorKind::InvalidMaxAttempts,
6572            ),
6573            (
6574                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
6575                    max_attempts: None,
6576                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
6577                    non_retryable_error_types: Vec::new(),
6578                }),
6579                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
6580            ),
6581            (
6582                ActivityOptions::new().retry_policy(
6583                    ActivityRetryPolicy::new(2)
6584                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
6585                ),
6586                ActivityOptionsErrorKind::TooManyBackoffIntervals,
6587            ),
6588            (
6589                ActivityOptions::new().retry_policy(
6590                    ActivityRetryPolicy::new(2).exponential_backoff(
6591                        Duration::from_secs(1),
6592                        0,
6593                        None,
6594                    ),
6595                ),
6596                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
6597            ),
6598            (
6599                ActivityOptions::new()
6600                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
6601                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
6602            ),
6603            (
6604                ActivityOptions::new().retry_policy(
6605                    ActivityRetryPolicy::new(10_002).exponential_backoff(
6606                        Duration::from_secs(1),
6607                        1,
6608                        None,
6609                    ),
6610                ),
6611                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
6612            ),
6613            (
6614                ActivityOptions::new().retry_policy(
6615                    ActivityRetryPolicy::new(2)
6616                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
6617                ),
6618                ActivityOptionsErrorKind::BackoffOverflow,
6619            ),
6620        ];
6621
6622        for (options, expected_kind) in cases {
6623            let ctx = workflow_context(Vec::new());
6624            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6625            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6626            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
6627                call.as_mut().poll(&mut task_context)
6628            else {
6629                panic!("expected typed activity validation error");
6630            };
6631            assert_eq!(error.kind, expected_kind);
6632            assert!(ctx.take_commands().expect("commands").is_empty());
6633        }
6634    }
6635
6636    #[test]
6637    fn activity_options_validate_positive_and_ordered_timeouts() {
6638        let zero_timeout_cases = [
6639            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
6640            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
6641            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
6642            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
6643        ];
6644        for options in zero_timeout_cases {
6645            assert_eq!(
6646                options.validate().expect_err("zero timeout").kind,
6647                ActivityOptionsErrorKind::TimeoutNotPositive
6648            );
6649        }
6650
6651        let ordering_cases = [
6652            ActivityOptions::new()
6653                .heartbeat_timeout(Duration::from_secs(11))
6654                .start_to_close_timeout(Duration::from_secs(10)),
6655            ActivityOptions::new()
6656                .start_to_close_timeout(Duration::from_secs(31))
6657                .schedule_to_close_timeout(Duration::from_secs(30)),
6658            ActivityOptions::new()
6659                .schedule_to_start_timeout(Duration::from_secs(31))
6660                .schedule_to_close_timeout(Duration::from_secs(30)),
6661        ];
6662        for options in ordering_cases {
6663            assert_eq!(
6664                options.validate().expect_err("timeout order").kind,
6665                ActivityOptionsErrorKind::TimeoutOrder
6666            );
6667        }
6668
6669        assert_eq!(
6670            ActivityOptions::new()
6671                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
6672                .validate()
6673                .expect_err("protocol integer overflow")
6674                .kind,
6675            ActivityOptionsErrorKind::TimeoutOverflow
6676        );
6677    }
6678
6679    #[test]
6680    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
6681        let ctx = workflow_context(completed_retry_activity_history());
6682        let mut call =
6683            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6684        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6685
6686        assert!(matches!(
6687            call.as_mut().poll(&mut task_context),
6688            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6689        ));
6690        assert!(ctx.take_commands().expect("commands").is_empty());
6691        ctx.ensure_history_consumed().expect("history consumed");
6692    }
6693
6694    #[test]
6695    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
6696        let mut options = retry_activity_options();
6697        options
6698            .retry_policy
6699            .as_mut()
6700            .expect("retry policy")
6701            .non_retryable_error_types
6702            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
6703
6704        let new_ctx = workflow_context(Vec::new());
6705        let mut new_call =
6706            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
6707        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6708        assert!(matches!(
6709            new_call.as_mut().poll(&mut task_context),
6710            Poll::Pending
6711        ));
6712        let commands = new_ctx.take_commands().expect("commands");
6713        assert_eq!(commands.len(), 1);
6714        assert_eq!(
6715            commands[0]["retry_policy"]["non_retryable_error_types"],
6716            json!(["PermanentError"])
6717        );
6718
6719        let replay_ctx = workflow_context(completed_retry_activity_history());
6720        let mut replay_call =
6721            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
6722        assert!(matches!(
6723            replay_call.as_mut().poll(&mut task_context),
6724            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6725        ));
6726        assert!(replay_ctx.take_commands().expect("commands").is_empty());
6727        replay_ctx
6728            .ensure_history_consumed()
6729            .expect("history consumed");
6730    }
6731
6732    #[test]
6733    fn replayed_intermediate_retry_remains_pending_across_restarts() {
6734        let history = completed_retry_activity_history()
6735            .into_iter()
6736            .take(3)
6737            .collect::<Vec<_>>();
6738
6739        for _restart in 0..2 {
6740            let ctx = workflow_context(history.clone());
6741            let mut call =
6742                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6743            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6744            assert!(matches!(
6745                call.as_mut().poll(&mut task_context),
6746                Poll::Pending
6747            ));
6748            assert!(ctx.take_commands().expect("commands").is_empty());
6749        }
6750    }
6751
6752    #[test]
6753    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
6754        let mut changed_queue = retry_activity_options();
6755        changed_queue.task_queue = Some("different-queue".to_string());
6756
6757        let mut changed_max_attempts = retry_activity_options();
6758        let retry_policy = changed_max_attempts
6759            .retry_policy
6760            .as_mut()
6761            .expect("retry policy");
6762        retry_policy.max_attempts = Some(4);
6763
6764        let mut changed_backoff = retry_activity_options();
6765        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
6766        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
6767            Duration::from_secs(3),
6768            Duration::from_secs(4),
6769        ]));
6770
6771        let mut changed_non_retryable_types = retry_activity_options();
6772        let retry_policy = changed_non_retryable_types
6773            .retry_policy
6774            .as_mut()
6775            .expect("retry policy");
6776        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
6777
6778        let mut changed_start_to_close = retry_activity_options();
6779        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
6780        let mut changed_schedule_to_start = retry_activity_options();
6781        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
6782        let mut changed_schedule_to_close = retry_activity_options();
6783        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
6784        let mut changed_heartbeat = retry_activity_options();
6785        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
6786
6787        let cases = [
6788            (changed_queue, "activity_task_queue_mismatch"),
6789            (changed_max_attempts, "activity_retry_policy_mismatch"),
6790            (changed_backoff, "activity_retry_policy_mismatch"),
6791            (
6792                changed_non_retryable_types,
6793                "activity_retry_policy_mismatch",
6794            ),
6795            (changed_start_to_close, "activity_retry_policy_mismatch"),
6796            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
6797            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
6798            (changed_heartbeat, "activity_retry_policy_mismatch"),
6799        ];
6800
6801        for (options, expected_reason) in cases {
6802            let ctx = workflow_context(completed_retry_activity_history());
6803            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
6804            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6805            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6806                call.as_mut().poll(&mut task_context)
6807            else {
6808                panic!("changed activity options must fail replay");
6809            };
6810            assert_eq!(failure.reason, expected_reason);
6811            assert_eq!(failure.sequence, Some(1));
6812            assert!(ctx.take_commands().expect("commands").is_empty());
6813        }
6814    }
6815
6816    #[test]
6817    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
6818        let cases = [
6819            (
6820                "execution_mode",
6821                json!("local"),
6822                "activity_execution_mode_mismatch",
6823            ),
6824            (
6825                "snapshot_version",
6826                json!(2),
6827                "activity_retry_policy_mismatch",
6828            ),
6829        ];
6830
6831        for (field, value, expected_reason) in cases {
6832            let mut history = completed_retry_activity_history();
6833            let activity = history[0].payload["activity"]
6834                .as_object_mut()
6835                .expect("activity snapshot");
6836            if field == "execution_mode" {
6837                activity.insert(field.to_string(), value);
6838            } else {
6839                activity["retry_policy"]
6840                    .as_object_mut()
6841                    .expect("retry snapshot")
6842                    .insert(field.to_string(), value);
6843            }
6844
6845            let ctx = workflow_context(history);
6846            let mut call =
6847                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6848            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6849            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6850                call.as_mut().poll(&mut task_context)
6851            else {
6852                panic!("changed {field} must fail replay");
6853            };
6854            assert_eq!(failure.reason, expected_reason);
6855            assert_eq!(failure.sequence, Some(1));
6856            assert!(ctx.take_commands().expect("commands").is_empty());
6857        }
6858    }
6859
6860    #[test]
6861    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
6862        let mut history = completed_retry_activity_history();
6863        let activity = history[0].payload["activity"]
6864            .as_object_mut()
6865            .expect("activity snapshot");
6866        activity.remove("execution_mode");
6867        activity.remove("retry_policy");
6868
6869        let mut current = retry_activity_options();
6870        current.start_to_close_timeout = Some(Duration::from_secs(45));
6871        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
6872        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
6873        current.heartbeat_timeout = Some(Duration::from_secs(12));
6874
6875        let ctx = workflow_context(history);
6876        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
6877        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6878        assert!(matches!(
6879            call.as_mut().poll(&mut task_context),
6880            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6881        ));
6882        assert!(ctx.take_commands().expect("commands").is_empty());
6883        ctx.ensure_history_consumed().expect("history consumed");
6884    }
6885
6886    #[test]
6887    fn terminal_activity_failed_after_start_returns_typed_failure() {
6888        let history = vec![
6889            history_event(
6890                "ActivityScheduled",
6891                json!({
6892                    "sequence": 1,
6893                    "activity_type": "flaky",
6894                    "activity_execution_id": "act-terminal",
6895                    "activity": {
6896                        "id": "act-terminal",
6897                        "sequence": 1,
6898                        "type": "flaky",
6899                        "queue": "critical-activities",
6900                        "retry_policy": {
6901                            "snapshot_version": 1,
6902                            "max_attempts": 3,
6903                            "backoff_seconds": [2, 4],
6904                            "non_retryable_error_types": ["PermanentError"]
6905                        }
6906                    }
6907                }),
6908            ),
6909            history_event(
6910                "ActivityStarted",
6911                json!({
6912                    "sequence": 1,
6913                    "activity_type": "flaky",
6914                    "activity_execution_id": "act-terminal",
6915                    "activity_attempt_id": "attempt-1",
6916                    "attempt_number": 1
6917                }),
6918            ),
6919            history_event(
6920                "ActivityFailed",
6921                json!({
6922                    "sequence": 1,
6923                    "activity_type": "flaky",
6924                    "activity_execution_id": "act-terminal",
6925                    "activity_attempt_id": "attempt-1",
6926                    "attempt_number": 1,
6927                    "failure_id": "failure-terminal",
6928                    "failure_category": "activity",
6929                    "exception_type": "PermanentError",
6930                    "message": "cannot retry",
6931                    "non_retryable": true
6932                }),
6933            ),
6934        ];
6935        let ctx = workflow_context(history);
6936        let mut call =
6937            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6938        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6939
6940        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6941            call.as_mut().poll(&mut task_context)
6942        else {
6943            panic!("terminal ActivityFailed must settle the activity future");
6944        };
6945        assert_eq!(failure.kind, ActivityFailureKind::Failed);
6946        assert_eq!(
6947            failure.activity_execution_id.as_deref(),
6948            Some("act-terminal")
6949        );
6950        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
6951        assert!(failure.non_retryable);
6952        assert!(ctx.take_commands().expect("commands").is_empty());
6953        ctx.ensure_history_consumed().expect("history consumed");
6954    }
6955
6956    #[test]
6957    fn activity_terminal_events_return_machine_readable_failures() {
6958        let cases = [
6959            (
6960                "ActivityFailed",
6961                json!({
6962                    "sequence": 1,
6963                    "activity_type": "charge-card",
6964                    "activity_execution_id": "act-1",
6965                    "activity_attempt_id": "attempt-2",
6966                    "attempt_number": 2,
6967                    "failure_id": "failure-1",
6968                    "failure_category": "activity",
6969                    "exception_type": "PaymentDeclined",
6970                    "exception_class": "payments.PaymentDeclined",
6971                    "message": "card declined",
6972                    "non_retryable": true
6973                }),
6974                ActivityFailureKind::Failed,
6975                "activity",
6976            ),
6977            (
6978                "ActivityCancelled",
6979                json!({
6980                    "sequence": 1,
6981                    "activity_type": "charge-card",
6982                    "activity_execution_id": "act-1",
6983                    "activity_attempt_id": "attempt-1"
6984                }),
6985                ActivityFailureKind::Cancelled,
6986                "cancelled",
6987            ),
6988        ];
6989
6990        for (event_type, payload, expected_kind, expected_reason) in cases {
6991            let ctx = workflow_context(vec![history_event(event_type, payload)]);
6992            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
6993            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6994            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6995                call.as_mut().poll(&mut task_context)
6996            else {
6997                panic!("expected terminal activity failure");
6998            };
6999            assert_eq!(failure.kind, expected_kind);
7000            assert_eq!(failure.reason, expected_reason);
7001            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
7002            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
7003        }
7004    }
7005
7006    #[test]
7007    fn every_activity_timeout_class_is_typed() {
7008        for timeout_kind in [
7009            "start_to_close",
7010            "schedule_to_start",
7011            "schedule_to_close",
7012            "heartbeat",
7013        ] {
7014            let ctx = workflow_context(vec![history_event(
7015                "ActivityTimedOut",
7016                json!({
7017                    "sequence": 1,
7018                    "activity_type": "slow",
7019                    "activity_execution_id": "act-timeout",
7020                    "activity_attempt_id": "attempt-timeout",
7021                    "failure_category": "timeout",
7022                    "timeout_kind": timeout_kind,
7023                    "message": "deadline expired"
7024                }),
7025            )]);
7026            let mut call = Box::pin(ctx.activity("slow", json!([])));
7027            let mut task_context = TaskContext::from_waker(noop_waker_ref());
7028            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
7029                call.as_mut().poll(&mut task_context)
7030            else {
7031                panic!("expected timeout failure");
7032            };
7033            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
7034            assert_eq!(failure.reason, timeout_kind);
7035            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
7036            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
7037        }
7038    }
7039
7040    #[test]
7041    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
7042        let ctx = workflow_context(Vec::new());
7043        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
7044        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7045
7046        assert!(matches!(
7047            sleep.as_mut().poll(&mut task_context),
7048            Poll::Pending
7049        ));
7050        assert!(matches!(
7051            sleep.as_mut().poll(&mut task_context),
7052            Poll::Pending
7053        ));
7054
7055        let commands = ctx.take_commands().expect("timer command");
7056        assert_eq!(
7057            commands,
7058            vec![json!({
7059                "type": "start_timer",
7060                "delay_seconds": 2,
7061            })]
7062        );
7063    }
7064
7065    #[test]
7066    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
7067        let history = vec![
7068            history_event(
7069                "TimerScheduled",
7070                json!({
7071                    "sequence": 1,
7072                    "timer_id": "timer-1",
7073                    "delay_seconds": 5,
7074                    "fire_at": "2026-07-11T12:00:05Z",
7075                }),
7076            ),
7077            history_event(
7078                "TimerFired",
7079                json!({
7080                    "sequence": 1,
7081                    "timer_id": "timer-1",
7082                    "delay_seconds": 5,
7083                    "fire_at": "2026-07-11T12:00:05Z",
7084                    "fired_at": "2026-07-11T12:00:05Z",
7085                }),
7086            ),
7087        ];
7088
7089        for _restart in 0..2 {
7090            let ctx = workflow_context(history.clone());
7091            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
7092            let mut task_context = TaskContext::from_waker(noop_waker_ref());
7093            assert!(matches!(
7094                sleep.as_mut().poll(&mut task_context),
7095                Poll::Ready(Ok(()))
7096            ));
7097            assert!(ctx.take_commands().expect("commands").is_empty());
7098            ctx.ensure_history_consumed().expect("history consumed");
7099        }
7100    }
7101
7102    #[test]
7103    fn workflow_sleep_rejects_changed_delay_during_replay() {
7104        let ctx = workflow_context(vec![
7105            history_event(
7106                "TimerScheduled",
7107                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7108            ),
7109            history_event(
7110                "TimerFired",
7111                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7112            ),
7113        ]);
7114        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
7115        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7116
7117        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7118            sleep.as_mut().poll(&mut task_context)
7119        else {
7120            panic!("changed timer delay must be rejected");
7121        };
7122        assert_eq!(failure.reason, "timer_delay_mismatch");
7123        assert_eq!(failure.sequence, Some(1));
7124    }
7125
7126    #[test]
7127    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
7128        let lone_fire = WorkflowState::new(
7129            vec![history_event(
7130                "TimerFired",
7131                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7132            )],
7133            "rust-workers".to_string(),
7134            JSON_CODEC.to_string(),
7135            None,
7136        )
7137        .expect_err("TimerFired requires TimerScheduled");
7138        assert!(matches!(
7139            lone_fire,
7140            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7141                if reason == "timer_schedule_missing_or_duplicate"
7142        ));
7143
7144        let wrong_identity = WorkflowState::new(
7145            vec![
7146                history_event(
7147                    "TimerScheduled",
7148                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7149                ),
7150                history_event(
7151                    "TimerFired",
7152                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7153                ),
7154            ],
7155            "rust-workers".to_string(),
7156            JSON_CODEC.to_string(),
7157            None,
7158        )
7159        .expect_err("fire must match scheduled timer identity");
7160        assert!(matches!(
7161            wrong_identity,
7162            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7163                if reason == "timer_identity_mismatch"
7164        ));
7165
7166        let duplicate_fire = WorkflowState::new(
7167            vec![
7168                history_event(
7169                    "TimerScheduled",
7170                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7171                ),
7172                history_event(
7173                    "TimerFired",
7174                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7175                ),
7176                history_event(
7177                    "TimerFired",
7178                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7179                ),
7180            ],
7181            "rust-workers".to_string(),
7182            JSON_CODEC.to_string(),
7183            None,
7184        )
7185        .expect_err("a durable timer cannot fire twice");
7186        assert!(matches!(
7187            duplicate_fire,
7188            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7189                if reason == "duplicate_timer_fire"
7190        ));
7191
7192        let wrong_fired_delay = WorkflowState::new(
7193            vec![
7194                history_event(
7195                    "TimerScheduled",
7196                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7197                ),
7198                history_event(
7199                    "TimerFired",
7200                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
7201                ),
7202            ],
7203            "rust-workers".to_string(),
7204            JSON_CODEC.to_string(),
7205            None,
7206        )
7207        .expect_err("timer schedule and fire delays must agree");
7208        assert!(matches!(
7209            wrong_fired_delay,
7210            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7211                if reason == "timer_history_delay_mismatch"
7212        ));
7213    }
7214
7215    #[test]
7216    fn replay_rejects_activity_moved_before_recorded_timer() {
7217        let ctx = workflow_context(vec![
7218            history_event(
7219                "TimerScheduled",
7220                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7221            ),
7222            history_event(
7223                "TimerFired",
7224                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7225            ),
7226            history_event(
7227                "ActivityCompleted",
7228                json!({
7229                    "sequence": 2,
7230                    "activity_type": "after-timer",
7231                    "payload_codec": "json",
7232                    "result": {"codec": "json", "blob": "\"done\""},
7233                }),
7234            ),
7235        ]);
7236        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
7237        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7238
7239        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7240            activity.as_mut().poll(&mut task_context)
7241        else {
7242            panic!("reordered durable command must be rejected");
7243        };
7244        assert_eq!(failure.reason, "recorded_command_mismatch");
7245        assert_eq!(failure.sequence, Some(1));
7246        assert_eq!(failure.expected.as_deref(), Some("timer"));
7247        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
7248    }
7249
7250    #[test]
7251    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
7252        let signal_then_timer = vec![
7253            history_event(
7254                "ConditionWaitOpened",
7255                json!({"sequence": 1, "condition_key": "signal:go"}),
7256            ),
7257            history_event(
7258                "SignalReceived",
7259                json!({
7260                    "signal_name": "go",
7261                    "arguments": ["now"],
7262                }),
7263            ),
7264            history_event(
7265                "TimerScheduled",
7266                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7267            ),
7268            history_event(
7269                "TimerFired",
7270                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7271            ),
7272        ];
7273
7274        let ctx = workflow_context(signal_then_timer.clone());
7275        let mut signal = Box::pin(ctx.wait_signal("go"));
7276        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7277        assert!(matches!(
7278            signal.as_mut().poll(&mut task_context),
7279            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
7280        ));
7281        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
7282        assert!(matches!(
7283            timer.as_mut().poll(&mut task_context),
7284            Poll::Ready(Ok(()))
7285        ));
7286        ctx.ensure_history_consumed()
7287            .expect("signal and timer history consumed in order");
7288
7289        let reordered = workflow_context(signal_then_timer);
7290        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
7291        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7292            timer_first.as_mut().poll(&mut task_context)
7293        else {
7294            panic!("timer cannot consume signal-wait-first history");
7295        };
7296        assert_eq!(failure.reason, "recorded_command_mismatch");
7297        assert_eq!(failure.sequence, Some(1));
7298        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
7299
7300        let timer_then_signal = vec![
7301            history_event(
7302                "TimerScheduled",
7303                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7304            ),
7305            history_event(
7306                "TimerFired",
7307                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7308            ),
7309            history_event(
7310                "ConditionWaitOpened",
7311                json!({"sequence": 2, "condition_key": "signal:go"}),
7312            ),
7313            history_event(
7314                "SignalReceived",
7315                json!({"signal_name": "go", "arguments": []}),
7316            ),
7317        ];
7318        let reordered = workflow_context(timer_then_signal);
7319        let mut signal_first = Box::pin(reordered.wait_signal("go"));
7320        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7321            signal_first.as_mut().poll(&mut task_context)
7322        else {
7323            panic!("signal wait cannot consume timer-first history");
7324        };
7325        assert_eq!(failure.reason, "recorded_command_mismatch");
7326        assert_eq!(failure.sequence, Some(1));
7327        assert_eq!(failure.expected.as_deref(), Some("timer"));
7328    }
7329
7330    #[test]
7331    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
7332        let duplicate_timer = WorkflowState::new(
7333            vec![
7334                history_event(
7335                    "TimerScheduled",
7336                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7337                ),
7338                history_event(
7339                    "TimerScheduled",
7340                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7341                ),
7342            ],
7343            "rust-workers".to_string(),
7344            JSON_CODEC.to_string(),
7345            None,
7346        )
7347        .expect_err("one workflow sequence cannot schedule two timers");
7348        assert!(matches!(
7349            duplicate_timer,
7350            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7351                if reason == "timer_schedule_missing_or_duplicate"
7352        ));
7353
7354        let colliding_kinds = WorkflowState::new(
7355            vec![
7356                history_event(
7357                    "TimerScheduled",
7358                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7359                ),
7360                history_event(
7361                    "ActivityCompleted",
7362                    json!({"sequence": 1, "activity_type": "same-sequence"}),
7363                ),
7364            ],
7365            "rust-workers".to_string(),
7366            JSON_CODEC.to_string(),
7367            None,
7368        )
7369        .expect_err("one workflow sequence cannot identify two command kinds");
7370        assert!(matches!(
7371            colliding_kinds,
7372            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7373                if reason == "durable_command_sequence_collision"
7374        ));
7375
7376        let duplicate_signal_wait = WorkflowState::new(
7377            vec![
7378                history_event(
7379                    "SignalWaitOpened",
7380                    json!({"sequence": 1, "signal_name": "go"}),
7381                ),
7382                history_event(
7383                    "SignalWaitOpened",
7384                    json!({"sequence": 1, "signal_name": "go"}),
7385                ),
7386            ],
7387            "rust-workers".to_string(),
7388            JSON_CODEC.to_string(),
7389            None,
7390        )
7391        .expect_err("one workflow sequence cannot open two signal waits");
7392        assert!(matches!(
7393            duplicate_signal_wait,
7394            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7395                if reason == "duplicate_signal_wait_open"
7396        ));
7397    }
7398
7399    #[test]
7400    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
7401        let ctx = workflow_context(Vec::new());
7402        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
7403        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7404        assert!(matches!(
7405            sleep.as_mut().poll(&mut task_context),
7406            Poll::Ready(Err(Error::TimerDurationOverflow))
7407        ));
7408        assert!(ctx.take_commands().expect("commands").is_empty());
7409    }
7410
7411    #[test]
7412    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
7413        let client = Client::new("http://127.0.0.1:8080").expect("client");
7414        let mut worker = Worker::new(client, "rust-workers");
7415        worker.register_workflow("rust.timer", |ctx, _input| async move {
7416            ctx.sleep(Duration::from_secs(5)).await?;
7417            ctx.activity("after-timer", json!([])).await
7418        });
7419
7420        let task = |history_events| WorkflowTask {
7421            task_id: "wft-rust-timer-1".to_string(),
7422            workflow_id: Some("wf-rust-timer".to_string()),
7423            run_id: Some("run-rust-timer".to_string()),
7424            workflow_type: "rust.timer".to_string(),
7425            payload_codec: JSON_CODEC.to_string(),
7426            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7427            history_events,
7428            total_history_events: None,
7429            next_history_page_token: None,
7430            workflow_task_attempt: 1,
7431            workflow_signal_id: None,
7432            signal_name: None,
7433            signal_arguments: None,
7434            lease_owner: Some("rust-worker".to_string()),
7435        };
7436
7437        let initial = worker
7438            .execute_workflow_task(task(Vec::new()))
7439            .expect("initial timer task");
7440        assert_eq!(
7441            initial,
7442            vec![json!({"type": "start_timer", "delay_seconds": 5})]
7443        );
7444
7445        let replayed = worker
7446            .execute_workflow_task(task(vec![
7447                history_event(
7448                    "TimerScheduled",
7449                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7450                ),
7451                history_event(
7452                    "TimerFired",
7453                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7454                ),
7455                history_event(
7456                    "ActivityCompleted",
7457                    json!({
7458                        "sequence": 2,
7459                        "activity_type": "after-timer",
7460                        "payload_codec": "json",
7461                        "result": {"codec": "json", "blob": "\"done\""},
7462                    }),
7463                ),
7464            ]))
7465            .expect("replayed workflow task");
7466        assert_eq!(replayed.len(), 1);
7467        assert_eq!(replayed[0]["type"], "complete_workflow");
7468        assert_eq!(
7469            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
7470            json!("done")
7471        );
7472    }
7473
7474    #[test]
7475    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
7476        let client = Client::new("http://127.0.0.1:8080").expect("client");
7477        let mut worker = Worker::new(client, "rust-workers");
7478        worker.register_workflow("rust.failing", |_ctx, _input| async move {
7479            Err(Error::Codec("rust_conformance_failure".to_string()))
7480        });
7481        let task = WorkflowTask {
7482            task_id: "wft-rust-failing-1".to_string(),
7483            workflow_id: Some("wf-rust-failing".to_string()),
7484            run_id: Some("run-rust-failing".to_string()),
7485            workflow_type: "rust.failing".to_string(),
7486            payload_codec: JSON_CODEC.to_string(),
7487            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
7488            history_events: Vec::new(),
7489            total_history_events: Some(0),
7490            next_history_page_token: None,
7491            workflow_task_attempt: 1,
7492            workflow_signal_id: None,
7493            signal_name: None,
7494            signal_arguments: None,
7495            lease_owner: Some("rust-worker".to_string()),
7496        };
7497
7498        let commands = worker
7499            .execute_workflow_task(task)
7500            .expect("handler failure becomes a workflow command");
7501
7502        assert_eq!(commands.len(), 1);
7503        assert_eq!(commands[0]["type"], "fail_workflow");
7504        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
7505        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
7506        assert_eq!(commands[0]["non_retryable"], false);
7507        assert_eq!(
7508            commands[0]["message"],
7509            "codec error: rust_conformance_failure"
7510        );
7511        assert_eq!(
7512            commands[0]["exception"]["message"],
7513            "codec error: rust_conformance_failure"
7514        );
7515    }
7516
7517    #[test]
7518    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
7519        let client = Client::new("http://127.0.0.1:8080").expect("client");
7520        let mut worker = Worker::new(client, "rust-workers");
7521        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
7522            ctx.sleep(Duration::from_secs(5)).await?;
7523            Ok(json!({"status": "timer fired"}))
7524        });
7525
7526        let task = WorkflowTask {
7527            task_id: "wft-rust-timer-pending".to_string(),
7528            workflow_id: Some("wf-rust-timer".to_string()),
7529            run_id: Some("run-rust-timer".to_string()),
7530            workflow_type: "rust.timer.pending".to_string(),
7531            payload_codec: JSON_CODEC.to_string(),
7532            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7533            history_events: vec![history_event(
7534                "TimerScheduled",
7535                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7536            )],
7537            total_history_events: Some(1),
7538            next_history_page_token: None,
7539            workflow_task_attempt: 1,
7540            workflow_signal_id: None,
7541            signal_name: None,
7542            signal_arguments: None,
7543            lease_owner: Some("rust-worker".to_string()),
7544        };
7545
7546        for _redelivery_or_restart in 0..2 {
7547            let commands = worker
7548                .execute_workflow_task(task.clone())
7549                .expect("recorded timer remains pending");
7550            assert!(
7551                commands.is_empty(),
7552                "recorded timer must not be rescheduled"
7553            );
7554        }
7555    }
7556
7557    #[test]
7558    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
7559        let client = Client::new("http://127.0.0.1:8080").expect("client");
7560        let mut worker = Worker::new(client, "rust-workers");
7561        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
7562            Ok(json!({"status": "completed"}))
7563        });
7564        let task = WorkflowTask {
7565            task_id: "wft-rust-timer-removed".to_string(),
7566            workflow_id: Some("wf-rust-timer".to_string()),
7567            run_id: Some("run-rust-timer".to_string()),
7568            workflow_type: "rust.timer.removed".to_string(),
7569            payload_codec: JSON_CODEC.to_string(),
7570            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7571            history_events: vec![
7572                history_event(
7573                    "TimerScheduled",
7574                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7575                ),
7576                history_event(
7577                    "TimerFired",
7578                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7579                ),
7580            ],
7581            total_history_events: Some(2),
7582            next_history_page_token: None,
7583            workflow_task_attempt: 1,
7584            workflow_signal_id: None,
7585            signal_name: None,
7586            signal_arguments: None,
7587            lease_owner: Some("rust-worker".to_string()),
7588        };
7589
7590        let Error::NonDeterministicReplay(failure) = worker
7591            .execute_workflow_task(task)
7592            .expect_err("removed timer must fail replay")
7593        else {
7594            panic!("expected typed replay failure");
7595        };
7596        assert_eq!(failure.reason, "recorded_commands_unconsumed");
7597        assert_eq!(failure.sequence, Some(1));
7598    }
7599
7600    #[test]
7601    fn workflow_context_emits_explicit_child_workflow_contract() {
7602        let ctx = WorkflowContext {
7603            state: Arc::new(Mutex::new(
7604                WorkflowState::new_with_identity(
7605                    Vec::new(),
7606                    Some("wf-parent".to_string()),
7607                    Some("run-parent".to_string()),
7608                    "parent-workers".to_string(),
7609                    JSON_CODEC.to_string(),
7610                    None,
7611                )
7612                .expect("workflow state"),
7613            )),
7614        };
7615        let options = ChildWorkflowOptions::new("python-workers")
7616            .parent_close_policy(ParentClosePolicy::RequestCancel)
7617            .retry_policy(ChildWorkflowRetryPolicy {
7618                max_attempts: Some(3),
7619                backoff_seconds: vec![1, 5],
7620                non_retryable_error_types: vec!["ValidationError".to_string()],
7621            })
7622            .execution_timeout_seconds(600)
7623            .run_timeout_seconds(120);
7624        let mut call = Box::pin(ctx.start_child_workflow(
7625            "python.fulfil-order",
7626            options,
7627            json!([{"order_id": "order-42"}]),
7628        ));
7629        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7630
7631        assert!(matches!(
7632            call.as_mut().poll(&mut task_context),
7633            Poll::Pending
7634        ));
7635        let commands = ctx.take_commands().expect("commands");
7636        assert_eq!(commands.len(), 1);
7637        let command = &commands[0];
7638        assert_eq!(command["type"], "start_child_workflow");
7639        assert_eq!(command["workflow_type"], "python.fulfil-order");
7640        assert_eq!(command["queue"], "python-workers");
7641        assert_eq!(command["parent_close_policy"], "request_cancel");
7642        assert_eq!(command["retry_policy"]["max_attempts"], 3);
7643        assert_eq!(command["execution_timeout_seconds"], 600);
7644        assert_eq!(command["run_timeout_seconds"], 120);
7645        assert_eq!(
7646            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
7647            json!([{"order_id": "order-42"}])
7648        );
7649    }
7650
7651    fn child_parent_worker() -> Worker {
7652        let client = Client::new("http://127.0.0.1:8080").expect("client");
7653        let mut worker = Worker::new(client, "rust-parent-workers");
7654        worker.register_workflow("rust.parent", |ctx, _input| async move {
7655            let child = ctx
7656                .start_child_workflow(
7657                    "python.child",
7658                    ChildWorkflowOptions::new("python-child-workers")
7659                        .parent_close_policy(ParentClosePolicy::Terminate),
7660                    json!([{"codec_probe": [1, true, "rust"]}]),
7661                )
7662                .await?;
7663            Ok(json!({
7664                "parent_workflow_id": child.parent.workflow_id,
7665                "parent_run_id": child.parent.run_id,
7666                "child_workflow_id": child.child.workflow_id,
7667                "child_run_id": child.child.run_id,
7668                "child_workflow_type": child.child_workflow_type,
7669                "result": child.result,
7670            }))
7671        });
7672        worker
7673    }
7674
7675    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
7676        WorkflowTask {
7677            task_id: "wft-child-parent".to_string(),
7678            workflow_id: Some("wf-parent".to_string()),
7679            run_id: Some("run-parent".to_string()),
7680            workflow_type: "rust.parent".to_string(),
7681            payload_codec: JSON_CODEC.to_string(),
7682            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
7683            history_events: vec![
7684                HistoryEvent {
7685                    event_type: "ChildWorkflowScheduled".to_string(),
7686                    payload: json!({
7687                        "sequence": 1,
7688                        "child_call_id": "call-child",
7689                        "child_workflow_instance_id": "wf-child",
7690                        "child_workflow_run_id": "run-child",
7691                        "child_workflow_type": "python.child",
7692                    }),
7693                    raw: HashMap::new(),
7694                },
7695                HistoryEvent {
7696                    event_type: event_type.to_string(),
7697                    payload,
7698                    raw: HashMap::new(),
7699                },
7700            ],
7701            total_history_events: Some(2),
7702            next_history_page_token: None,
7703            workflow_task_attempt: 1,
7704            workflow_signal_id: None,
7705            signal_name: None,
7706            signal_arguments: None,
7707            lease_owner: Some("rust-worker".to_string()),
7708        }
7709    }
7710
7711    #[test]
7712    fn committed_child_result_replays_without_starting_a_duplicate() {
7713        let worker = child_parent_worker();
7714        let task = child_parent_task(
7715            "ChildRunCompleted",
7716            json!({
7717                "sequence": 1,
7718                "child_call_id": "call-child",
7719                "child_workflow_instance_id": "wf-child",
7720                "child_workflow_run_id": "run-child",
7721                "child_workflow_type": "python.child",
7722                "payload_codec": "json",
7723                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
7724            }),
7725        );
7726
7727        for _restart in 0..2 {
7728            let commands = worker
7729                .execute_workflow_task(task.clone())
7730                .expect("replayed parent task");
7731            assert_eq!(commands.len(), 1);
7732            assert_eq!(commands[0]["type"], "complete_workflow");
7733            assert!(!commands
7734                .iter()
7735                .any(|command| command["type"] == "start_child_workflow"));
7736            let output =
7737                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7738            assert_eq!(output["parent_workflow_id"], "wf-parent");
7739            assert_eq!(output["parent_run_id"], "run-parent");
7740            assert_eq!(output["child_workflow_id"], "wf-child");
7741            assert_eq!(output["child_run_id"], "run-child");
7742            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
7743        }
7744    }
7745
7746    #[test]
7747    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
7748        let worker = child_parent_worker();
7749        let mut task = child_parent_task("unused", Value::Null);
7750        task.history_events.truncate(1);
7751        task.total_history_events = Some(1);
7752
7753        for _redelivery_or_restart in 0..2 {
7754            let commands = worker
7755                .execute_workflow_task(task.clone())
7756                .expect("recorded child remains pending");
7757            assert!(
7758                commands.is_empty(),
7759                "recorded pending child must not be started again"
7760            );
7761        }
7762    }
7763
7764    #[test]
7765    fn child_cancellation_becomes_stable_parent_failure_command() {
7766        let worker = child_parent_worker();
7767        let task = child_parent_task(
7768            "ChildRunCancelled",
7769            json!({
7770                "sequence": 1,
7771                "child_workflow_instance_id": "wf-child",
7772                "child_workflow_run_id": "run-child",
7773                "child_workflow_type": "python.child",
7774                "failure_id": "failure-child",
7775                "failure_category": "cancelled",
7776                "message": "cancelled by parent-close policy",
7777            }),
7778        );
7779
7780        let commands = worker
7781            .execute_workflow_task(task)
7782            .expect("parent settlement");
7783        assert_eq!(commands.len(), 1);
7784        assert_eq!(commands[0]["type"], "fail_workflow");
7785        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
7786        assert_eq!(
7787            commands[0]["exception"]["properties"]["reason"],
7788            "cancelled"
7789        );
7790        assert_eq!(
7791            commands[0]["exception"]["properties"]["child_workflow_run_id"],
7792            "run-child"
7793        );
7794    }
7795
7796    #[test]
7797    fn workflow_can_handle_typed_child_failure() {
7798        let client = Client::new("http://127.0.0.1:8080").expect("client");
7799        let mut worker = Worker::new(client, "rust-parent-workers");
7800        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
7801            match ctx
7802                .start_child_workflow(
7803                    "python.child",
7804                    ChildWorkflowOptions::new("python-child-workers"),
7805                    json!([]),
7806                )
7807                .await
7808            {
7809                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
7810                    "reason": failure.reason,
7811                    "failure_id": failure.failure_id,
7812                    "exception_class": failure.exception_class,
7813                    "child_run_id": failure.child_workflow_run_id,
7814                })),
7815                Err(error) => Err(error),
7816                Ok(_) => Err(Error::WorkerLoop(
7817                    "child unexpectedly succeeded".to_string(),
7818                )),
7819            }
7820        });
7821        let mut task = child_parent_task(
7822            "ChildRunFailed",
7823            json!({
7824                "sequence": 1,
7825                "child_workflow_instance_id": "wf-child",
7826                "child_workflow_run_id": "run-child",
7827                "child_workflow_type": "python.child",
7828                "failure_id": "failure-child",
7829                "failure_category": "child_workflow",
7830                "message": "payment rejected",
7831                "exception": {
7832                    "type": "PaymentRejected",
7833                    "class": "payments.PaymentRejected",
7834                    "message": "payment rejected"
7835                }
7836            }),
7837        );
7838        task.workflow_type = "rust.handled-parent".to_string();
7839
7840        let commands = worker.execute_workflow_task(task).expect("handled failure");
7841        assert_eq!(commands[0]["type"], "complete_workflow");
7842        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7843        assert_eq!(output["reason"], "child_workflow");
7844        assert_eq!(output["failure_id"], "failure-child");
7845        assert_eq!(output["exception_class"], "payments.PaymentRejected");
7846        assert_eq!(output["child_run_id"], "run-child");
7847    }
7848
7849    #[test]
7850    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
7851        let client = Client::new("http://127.0.0.1:8080").expect("client");
7852        let mut worker = Worker::new(client, "rust-workers");
7853
7854        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
7855            let signal = ctx.wait_signal("start").await?;
7856            let name = signal
7857                .first()
7858                .and_then(|value| value.as_str())
7859                .unwrap_or("world");
7860            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
7861            Ok(json!({
7862                "greeting": greeting,
7863                "language": "rust"
7864            }))
7865        });
7866
7867        let signal_arguments =
7868            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
7869        let task = WorkflowTask {
7870            task_id: "wft-rust-signal-1".to_string(),
7871            workflow_id: Some("wf-rust-hello".to_string()),
7872            run_id: Some("run-rust-hello".to_string()),
7873            workflow_type: "rust.hello_workflow".to_string(),
7874            payload_codec: DEFAULT_CODEC.to_string(),
7875            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7876            history_events: vec![HistoryEvent {
7877                event_type: "SignalReceived".to_string(),
7878                payload: json!({
7879                    "signal_id": "sig-rust-1",
7880                    "signal_name": "start"
7881                }),
7882                raw: HashMap::new(),
7883            }],
7884            total_history_events: Some(1),
7885            next_history_page_token: None,
7886            workflow_task_attempt: 1,
7887            workflow_signal_id: Some("sig-rust-1".to_string()),
7888            signal_name: Some("start".to_string()),
7889            signal_arguments: Some(signal_arguments),
7890            lease_owner: Some("rust-worker".to_string()),
7891        };
7892
7893        let commands = worker.execute_workflow_task(task).expect("workflow task");
7894
7895        assert_eq!(commands.len(), 1);
7896        assert_eq!(commands[0]["type"], "schedule_activity");
7897        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
7898        assert_eq!(
7899            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
7900            json!(["Rust"])
7901        );
7902    }
7903
7904    #[test]
7905    fn workflow_task_appends_paginated_history_events() {
7906        let mut task = WorkflowTask {
7907            task_id: "wft-rust-pages-1".to_string(),
7908            workflow_id: Some("wf-rust-pages".to_string()),
7909            run_id: Some("run-rust-pages".to_string()),
7910            workflow_type: "rust.hello_workflow".to_string(),
7911            payload_codec: DEFAULT_CODEC.to_string(),
7912            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7913            history_events: vec![HistoryEvent {
7914                event_type: "WorkflowStarted".to_string(),
7915                payload: json!({}),
7916                raw: HashMap::new(),
7917            }],
7918            total_history_events: Some(3),
7919            next_history_page_token: Some("MQ==".to_string()),
7920            workflow_task_attempt: 1,
7921            workflow_signal_id: None,
7922            signal_name: None,
7923            signal_arguments: None,
7924            lease_owner: Some("rust-worker".to_string()),
7925        };
7926
7927        task.append_history_page(WorkflowTaskHistoryPage {
7928            history_events: vec![
7929                HistoryEvent {
7930                    event_type: "SignalReceived".to_string(),
7931                    payload: json!({
7932                        "signal_id": "sig-rust-1",
7933                        "signal_name": "start",
7934                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
7935                            .expect("signal arguments")
7936                    }),
7937                    raw: HashMap::new(),
7938                },
7939                HistoryEvent {
7940                    event_type: "MarkerRecorded".to_string(),
7941                    payload: json!({"sequence": 3}),
7942                    raw: HashMap::new(),
7943                },
7944            ],
7945            total_history_events: Some(3),
7946            next_history_page_token: None,
7947        });
7948
7949        assert_eq!(task.history_events.len(), 3);
7950        assert_eq!(task.total_history_events, Some(3));
7951        assert_eq!(task.next_history_page_token, None);
7952
7953        let signals =
7954            signal_values(&task.history_events, "start", DEFAULT_CODEC, None).expect("signals");
7955        assert_eq!(signals, vec![vec![json!("Rust")]]);
7956    }
7957
7958    #[tokio::test]
7959    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
7960        let client = Client::new("http://127.0.0.1:8080").expect("client");
7961        let mut worker = Worker::new(client, "rust-workers");
7962        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
7963        worker.register_query("counter", "current", |ctx, _args| async move {
7964            let mut count = 0_i64;
7965            for signal in ctx.signal_events() {
7966                let value = signal
7967                    .arguments
7968                    .first()
7969                    .and_then(Value::as_i64)
7970                    .unwrap_or_default();
7971                match signal.name.as_str() {
7972                    "increment" => count += value,
7973                    "set" => count = value,
7974                    _ => {}
7975                }
7976            }
7977            Ok(json!(count))
7978        });
7979
7980        let task = QueryTask {
7981            query_task_id: "query-rust-counter".to_string(),
7982            query_task_attempt: 1,
7983            lease_owner: Some("rust-worker".to_string()),
7984            workflow_id: Some("counter-1".to_string()),
7985            run_id: Some("run-counter-1".to_string()),
7986            workflow_type: "counter".to_string(),
7987            query_name: "current".to_string(),
7988            payload_codec: DEFAULT_CODEC.to_string(),
7989            workflow_arguments: Some(
7990                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
7991            ),
7992            query_arguments: Some(
7993                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
7994            ),
7995            history_events: vec![
7996                HistoryEvent {
7997                    event_type: "SignalReceived".to_string(),
7998                    payload: json!({
7999                        "signal_id": "php-signal-1",
8000                        "signal_name": "increment",
8001                        "workflow_sequence": 1,
8002                        "payload_codec": DEFAULT_CODEC,
8003                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
8004                    }),
8005                    raw: HashMap::new(),
8006                },
8007                HistoryEvent {
8008                    event_type: "SignalReceived".to_string(),
8009                    payload: json!({
8010                        "signal_id": "python-signal-2",
8011                        "signal_name": "increment",
8012                        "workflow_sequence": 2,
8013                        "payload_codec": JSON_CODEC,
8014                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
8015                    }),
8016                    raw: HashMap::new(),
8017                },
8018                HistoryEvent {
8019                    event_type: "SignalReceived".to_string(),
8020                    payload: json!({
8021                        "signal_id": "rust-signal-3",
8022                        "signal_name": "set",
8023                        "workflow_sequence": 3,
8024                        "payload_codec": DEFAULT_CODEC,
8025                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
8026                    }),
8027                    raw: HashMap::new(),
8028                },
8029            ],
8030            history_export: None,
8031            run_status: Some("completed".to_string()),
8032        };
8033
8034        let result = worker.execute_query_task(task).await.expect("query result");
8035        assert_eq!(result, json!(0));
8036    }
8037
8038    #[tokio::test]
8039    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
8040        let worker = replay_counter_worker();
8041        let running_history = json!([
8042            {
8043                "type": "ActivityCompleted",
8044                "payload": {
8045                    "sequence": 1,
8046                    "activity_type": "load-counter",
8047                    "payload_codec": "json",
8048                    "result": {"codec": "json", "blob": "\"loaded\""}
8049                }
8050            },
8051            {
8052                "type": "SignalReceived",
8053                "payload": {
8054                    "signal_id": "signal-3",
8055                    "signal_name": "increment",
8056                    "payload_codec": "json",
8057                    "arguments": {"codec": "json", "blob": "[3]"}
8058                }
8059            }
8060        ]);
8061
8062        let running = worker
8063            .execute_query_task(replay_counter_query(
8064                "current",
8065                running_history.clone(),
8066                "running",
8067            ))
8068            .await
8069            .expect("running replay query");
8070        assert_eq!(
8071            running,
8072            json!({"loaded": "loaded", "count": 3, "finished": false})
8073        );
8074
8075        let detached = worker
8076            .execute_query_task(replay_counter_query(
8077                "detached-mutation",
8078                running_history.clone(),
8079                "running",
8080            ))
8081            .await
8082            .expect("query mutates only its detached state clone");
8083        assert_eq!(detached, json!(999));
8084        let failed = worker
8085            .execute_query_task(replay_counter_query(
8086                "failed-mutation",
8087                running_history.clone(),
8088                "running",
8089            ))
8090            .await
8091            .expect_err("failed query");
8092        assert_eq!(failed.reason, "query_rejected");
8093        let unchanged = worker
8094            .execute_query_task(replay_counter_query("current", running_history, "running"))
8095            .await
8096            .expect("later query reconstructs unchanged state");
8097        assert_eq!(unchanged, running);
8098
8099        let restarted_worker = replay_counter_worker();
8100        let restarted_task: QueryTask = serde_json::from_value(json!({
8101            "query_task_id": "query-after-restart",
8102            "workflow_id": "counter-1",
8103            "run_id": "run-counter-1",
8104            "workflow_type": "replay-counter",
8105            "query_name": "current",
8106            "payload_codec": "json",
8107            "workflow_arguments": {"codec": "json", "blob": "[]"},
8108            "query_arguments": {"codec": "json", "blob": "[]"},
8109            "history_events": [],
8110            "history_export": {
8111                "payloads": {"codec": "json"},
8112                "history_events": [
8113                    {
8114                        "type": "ActivityCompleted",
8115                        "payload": {
8116                            "sequence": 1,
8117                            "activity_type": "load-counter",
8118                            "payload_codec": "json",
8119                            "result": null
8120                        }
8121                    },
8122                    {
8123                        "type": "SignalReceived",
8124                        "payload": {"signal_id": "signal-3", "signal_name": "increment"}
8125                    },
8126                    {
8127                        "type": "SignalReceived",
8128                        "payload": {"signal_id": "signal-5", "signal_name": "increment"}
8129                    }
8130                ],
8131                "activities": [{
8132                    "sequence": 1,
8133                    "activity_type": "load-counter",
8134                    "payload_codec": "json",
8135                    "result": {"codec": "json", "blob": "\"loaded\""}
8136                }],
8137                "signals": [
8138                    {
8139                        "id": "signal-3",
8140                        "name": "increment",
8141                        "payload_codec": "json",
8142                        "arguments": "[3]"
8143                    },
8144                    {
8145                        "id": "signal-5",
8146                        "name": "increment",
8147                        "payload_codec": "json",
8148                        "arguments": "[5]"
8149                    }
8150                ]
8151            },
8152            "run_status": "completed"
8153        }))
8154        .expect("cold replay query task");
8155        let completed = restarted_worker
8156            .execute_query_task(restarted_task)
8157            .await
8158            .expect("completed cold replay query");
8159        assert_eq!(
8160            completed,
8161            json!({"loaded": "loaded", "count": 8, "finished": true})
8162        );
8163    }
8164
8165    #[tokio::test]
8166    async fn replayed_query_replay_failures_are_machine_readable() {
8167        let worker = replay_counter_worker();
8168        let task = replay_counter_query(
8169            "current",
8170            json!([{
8171                "type": "ActivityCompleted",
8172                "payload": {
8173                    "sequence": 1,
8174                    "payload_codec": "json",
8175                    "result": {"codec": "json", "blob": "{"}
8176                }
8177            }]),
8178            "running",
8179        );
8180        let failure = worker
8181            .execute_query_task(task)
8182            .await
8183            .expect_err("invalid replay history payload");
8184        assert_eq!(failure.reason, "query_workflow_state_unavailable");
8185        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
8186    }
8187
8188    #[tokio::test]
8189    async fn query_task_restores_compact_history_from_export() {
8190        let client = Client::new("http://127.0.0.1:8080").expect("client");
8191        let mut worker = Worker::new(client, "rust-workers");
8192        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8193        worker.register_query("counter", "current", |ctx, _args| async move {
8194            Ok(json!(ctx.signals("increment")[0][0]))
8195        });
8196        let task: QueryTask = serde_json::from_value(json!({
8197            "query_task_id": "query-export",
8198            "workflow_type": "counter",
8199            "query_name": "current",
8200            "payload_codec": "json",
8201            "workflow_arguments": {"codec": "json", "blob": "[]"},
8202            "query_arguments": {"codec": "json", "blob": "[]"},
8203            "history_events": [],
8204            "history_export": {
8205                "payloads": {"codec": "json"},
8206                "history_events": [{
8207                    "type": "SignalReceived",
8208                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
8209                }],
8210                "signals": [{
8211                    "id": "signal-export",
8212                    "name": "increment",
8213                    "status": "applied",
8214                    "workflow_sequence": 1,
8215                    "payload_codec": "json",
8216                    "arguments": "[9]"
8217                }]
8218            }
8219        }))
8220        .expect("query task");
8221
8222        let result = worker.execute_query_task(task).await.expect("query result");
8223        assert_eq!(result, json!(9));
8224    }
8225
8226    #[tokio::test]
8227    async fn query_task_failures_have_stable_reasons() {
8228        let client = Client::new("http://127.0.0.1:8080").expect("client");
8229        let mut worker = Worker::new(client, "rust-workers");
8230        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8231        worker.register_query(
8232            "counter",
8233            "current",
8234            |_ctx, _args| async move { Ok(json!(0)) },
8235        );
8236
8237        let base_task = QueryTask {
8238            query_task_id: "query-errors".to_string(),
8239            query_task_attempt: 1,
8240            lease_owner: None,
8241            workflow_id: Some("counter-errors".to_string()),
8242            run_id: Some("run-errors".to_string()),
8243            workflow_type: "counter".to_string(),
8244            query_name: "missing".to_string(),
8245            payload_codec: JSON_CODEC.to_string(),
8246            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8247            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8248            history_events: Vec::new(),
8249            history_export: None,
8250            run_status: Some("running".to_string()),
8251        };
8252
8253        let unknown = worker
8254            .execute_query_task(base_task.clone())
8255            .await
8256            .expect_err("unknown query");
8257        assert_eq!(unknown.reason, "rejected_unknown_query");
8258
8259        let mut malformed = base_task;
8260        malformed.query_name = "current".to_string();
8261        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
8262        let malformed = worker
8263            .execute_query_task(malformed)
8264            .await
8265            .expect_err("malformed payload");
8266        assert_eq!(malformed.reason, "query_payload_decode_failed");
8267
8268        let client = Client::new("http://127.0.0.1:8080").expect("client");
8269        let mut unavailable_worker = Worker::new(client, "rust-workers");
8270        unavailable_worker
8271            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8272        let unavailable_task: QueryTask = serde_json::from_value(json!({
8273            "query_task_id": "query-unavailable",
8274            "workflow_type": "counter",
8275            "query_name": "current",
8276            "payload_codec": "json",
8277            "workflow_arguments": {"codec": "json", "blob": "[]"},
8278            "query_arguments": {"codec": "json", "blob": "[]"}
8279        }))
8280        .expect("query task");
8281        let unavailable = unavailable_worker
8282            .execute_query_task(unavailable_task)
8283            .await
8284            .expect_err("query handler unavailable");
8285        assert_eq!(unavailable.reason, "query_handler_unavailable");
8286    }
8287
8288    #[tokio::test]
8289    async fn client_query_decodes_result_and_typed_failure() {
8290        let server = MockWorkerServer::start();
8291        let client = Client::builder(server.base_url())
8292            .timeout(Duration::from_secs(2))
8293            .build()
8294            .expect("client");
8295
8296        let result = client
8297            .query_workflow("counter-1", "current", json!([]))
8298            .await
8299            .expect("query result");
8300        assert_eq!(result, json!({"count": 8}));
8301
8302        let error = client
8303            .query_workflow("counter-1", "missing", json!([]))
8304            .await
8305            .expect_err("unknown query");
8306        let Error::QueryFailed(failure) = error else {
8307            panic!("expected typed query failure");
8308        };
8309        assert_eq!(failure.status, 404);
8310        assert_eq!(failure.reason, "rejected_unknown_query");
8311    }
8312
8313    #[tokio::test]
8314    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
8315        let server = MockWorkerServer::start();
8316        let client = Client::builder(server.base_url())
8317            .timeout(Duration::from_secs(2))
8318            .build()
8319            .expect("client");
8320
8321        let options = WorkflowCommandOptions::new()
8322            .reason("cleanup requested")
8323            .request_id("cancel-17");
8324        let cancelled = client
8325            .cancel_workflow("wf-lifecycle", options)
8326            .await
8327            .expect("instance cancellation");
8328        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
8329        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
8330        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
8331        assert_eq!(
8332            server.request_body("/api/workflows/wf-lifecycle/cancel"),
8333            json!({"reason":"cleanup requested","request_id":"cancel-17"})
8334        );
8335
8336        let terminated = client
8337            .terminate_workflow(
8338                "wf-lifecycle",
8339                WorkflowCommandOptions::new().reason("forced stop"),
8340            )
8341            .await
8342            .expect("instance termination");
8343        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
8344        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
8345
8346        client
8347            .cancel_workflow_run(
8348                "wf-lifecycle",
8349                "run-current",
8350                WorkflowCommandOptions::default(),
8351            )
8352            .await
8353            .expect("selected run cancellation");
8354        client
8355            .terminate_workflow_run(
8356                "wf-lifecycle",
8357                "run-current",
8358                WorkflowCommandOptions::default(),
8359            )
8360            .await
8361            .expect("selected run termination");
8362
8363        for (command, error) in [
8364            (
8365                WorkflowCommandKind::Cancel,
8366                client
8367                    .cancel_workflow_run(
8368                        "wf-lifecycle",
8369                        "run-stale",
8370                        WorkflowCommandOptions::default(),
8371                    )
8372                    .await
8373                    .expect_err("stale cancellation must be rejected"),
8374            ),
8375            (
8376                WorkflowCommandKind::Terminate,
8377                client
8378                    .terminate_workflow_run(
8379                        "wf-lifecycle",
8380                        "run-stale",
8381                        WorkflowCommandOptions::default(),
8382                    )
8383                    .await
8384                    .expect_err("stale termination must be rejected"),
8385            ),
8386        ] {
8387            let Error::WorkflowCommandRejected(rejection) = error else {
8388                panic!("expected typed command rejection");
8389            };
8390            assert_eq!(rejection.command, command);
8391            assert_eq!(rejection.status, 409);
8392            assert_eq!(rejection.reason, "historical_run_command_rejected");
8393            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
8394            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
8395        }
8396    }
8397
8398    #[tokio::test]
8399    async fn workflow_start_options_send_server_enforced_deadlines() {
8400        let server = MockWorkerServer::start();
8401        let client = Client::builder(server.base_url())
8402            .timeout(Duration::from_secs(2))
8403            .build()
8404            .expect("client");
8405
8406        let handle = client
8407            .start_workflow_with_options(
8408                "rust.timeout",
8409                "rust-timeouts",
8410                "wf-start-options",
8411                WorkflowStartOptions::new()
8412                    .execution_timeout_seconds(30)
8413                    .run_timeout_seconds(1),
8414                json!([]),
8415            )
8416            .await
8417            .expect("workflow start");
8418
8419        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
8420        let body = server.request_body("/api/workflows");
8421        assert_eq!(body["execution_timeout_seconds"], 30);
8422        assert_eq!(body["run_timeout_seconds"], 1);
8423
8424        let invalid = client
8425            .start_workflow_with_options(
8426                "rust.timeout",
8427                "rust-timeouts",
8428                "wf-invalid-options",
8429                WorkflowStartOptions::new()
8430                    .execution_timeout_seconds(1)
8431                    .run_timeout_seconds(2),
8432                json!([]),
8433            )
8434            .await
8435            .expect_err("invalid deadline ordering");
8436        assert!(invalid
8437            .to_string()
8438            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
8439    }
8440
8441    #[tokio::test]
8442    async fn workflow_result_returns_each_typed_terminal_outcome() {
8443        let server = MockWorkerServer::start();
8444        let client = Client::builder(server.base_url())
8445            .timeout(Duration::from_secs(2))
8446            .build()
8447            .expect("client");
8448        let options = WorkflowResultOptions {
8449            poll_interval: Duration::ZERO,
8450            timeout: Duration::from_secs(1),
8451        };
8452
8453        let failed = WorkflowHandle {
8454            client: client.clone(),
8455            workflow_id: "wf-failed".to_string(),
8456            run_id: Some("run-failed".to_string()),
8457            workflow_type: "failure".to_string(),
8458        }
8459        .result(options)
8460        .await
8461        .expect_err("failed outcome");
8462        let Error::WorkflowFailed(failure) = failed else {
8463            panic!("expected WorkflowFailed");
8464        };
8465        assert_eq!(failure.workflow_id, "wf-failed");
8466        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
8467        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
8468        assert_eq!(failure.failure_category.as_deref(), Some("application"));
8469        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
8470        assert_eq!(
8471            failure.exception_class.as_deref(),
8472            Some("billing::PaymentError")
8473        );
8474        assert_eq!(failure.non_retryable, Some(true));
8475
8476        for (workflow_id, expected_kind, expected_reason) in [
8477            (
8478                "wf-cancelled",
8479                WorkflowTerminalKind::Cancelled,
8480                "cleanup requested",
8481            ),
8482            (
8483                "wf-terminated",
8484                WorkflowTerminalKind::Terminated,
8485                "forced stop",
8486            ),
8487            (
8488                "wf-timed-out",
8489                WorkflowTerminalKind::TimedOut,
8490                "run_timeout",
8491            ),
8492        ] {
8493            let error = WorkflowHandle {
8494                client: client.clone(),
8495                workflow_id: workflow_id.to_string(),
8496                run_id: None,
8497                workflow_type: "terminal".to_string(),
8498            }
8499            .result(options)
8500            .await
8501            .expect_err("typed terminal outcome");
8502            let outcome = match error {
8503                Error::WorkflowCancelled(outcome) => outcome,
8504                Error::WorkflowTerminated(outcome) => outcome,
8505                Error::WorkflowTimedOut(outcome) => outcome,
8506                other => panic!("unexpected terminal error: {other}"),
8507            };
8508            assert_eq!(outcome.kind, expected_kind);
8509            assert_eq!(outcome.workflow_id, workflow_id);
8510            assert_eq!(outcome.reason, expected_reason);
8511        }
8512
8513        let wait_timeout = WorkflowHandle {
8514            client,
8515            workflow_id: "wf-waiting".to_string(),
8516            run_id: Some("run-waiting".to_string()),
8517            workflow_type: "waiting".to_string(),
8518        }
8519        .result(WorkflowResultOptions {
8520            poll_interval: Duration::ZERO,
8521            timeout: Duration::ZERO,
8522        })
8523        .await
8524        .expect_err("client wait timeout");
8525        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
8526            panic!("expected typed client timeout");
8527        };
8528        assert_eq!(timeout.reason, "result_wait_timeout");
8529        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
8530        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
8531    }
8532
8533    #[tokio::test]
8534    async fn workflow_result_awaits_the_handle_selected_run() {
8535        let server = MockWorkerServer::start();
8536        let client = Client::builder(server.base_url())
8537            .timeout(Duration::from_secs(2))
8538            .build()
8539            .expect("client");
8540
8541        let error = WorkflowHandle {
8542            client,
8543            workflow_id: "wf-selected".to_string(),
8544            run_id: Some("run-selected".to_string()),
8545            workflow_type: "selected".to_string(),
8546        }
8547        .result(WorkflowResultOptions {
8548            poll_interval: Duration::ZERO,
8549            timeout: Duration::from_secs(1),
8550        })
8551        .await
8552        .expect_err("the selected run is cancelled even though the current run completed");
8553
8554        let Error::WorkflowCancelled(outcome) = error else {
8555            panic!("expected selected run cancellation");
8556        };
8557        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
8558        assert_eq!(outcome.reason, "selected run cancelled");
8559        assert_eq!(
8560            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
8561            1
8562        );
8563        assert_eq!(server.request_count("/api/workflows/wf-selected"), 0);
8564    }
8565
8566    #[tokio::test]
8567    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
8568        let server = MockWorkerServer::draining_polls();
8569        let client = Client::builder(server.base_url())
8570            .timeout(Duration::from_secs(2))
8571            .build()
8572            .expect("client");
8573
8574        let workflow = client
8575            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
8576            .await
8577            .expect("workflow drain response");
8578        let activity = client
8579            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
8580            .await
8581            .expect("activity drain response");
8582        let query = client
8583            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
8584            .await
8585            .expect("query drain response");
8586
8587        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
8588            assert_eq!(
8589                outcome,
8590                WorkerPollOutcome::Stop {
8591                    poll_status: Some("draining".to_string()),
8592                    reason: Some("worker_draining".to_string()),
8593                }
8594            );
8595        }
8596
8597        assert!(client
8598            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
8599            .await
8600            .expect("compatibility poll")
8601            .is_none());
8602    }
8603
8604    #[tokio::test]
8605    async fn managed_worker_honors_drain_stop_for_every_task_family() {
8606        let server = MockWorkerServer::draining_polls();
8607        let client = Client::builder(server.base_url())
8608            .timeout(Duration::from_secs(2))
8609            .build()
8610            .expect("client");
8611
8612        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
8613            .worker_id("draining-workflow-worker")
8614            .poll_timeout(Duration::ZERO);
8615        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
8616        workflow_worker
8617            .run()
8618            .await
8619            .expect("workflow drain is a clean stop");
8620
8621        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
8622            .worker_id("draining-activity-worker")
8623            .poll_timeout(Duration::ZERO);
8624        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
8625        activity_worker
8626            .run()
8627            .await
8628            .expect("activity drain is a clean stop");
8629
8630        let mut query_worker = Worker::new(client, "rust-workers")
8631            .worker_id("draining-query-worker")
8632            .poll_timeout(Duration::ZERO);
8633        query_worker.register_query("counter", "current", |_ctx, _args| async {
8634            Ok(Value::Null)
8635        });
8636        query_worker
8637            .run()
8638            .await
8639            .expect("query drain is a clean stop");
8640    }
8641
8642    #[tokio::test]
8643    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
8644        let server = MockWorkerServer::start();
8645        let client = Client::builder(server.base_url())
8646            .timeout(Duration::from_secs(2))
8647            .build()
8648            .expect("client");
8649
8650        let heartbeat = client
8651            .heartbeat_activity_task(
8652                "activity-cancel",
8653                "attempt-cancel",
8654                "rust-worker",
8655                json!({"stage":"cleanup"}),
8656            )
8657            .await
8658            .expect("cancellation heartbeat");
8659        assert!(heartbeat.cancel_requested);
8660        assert!(heartbeat.should_stop());
8661        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
8662        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
8663
8664        let error = client
8665            .complete_activity_task(
8666                "activity-cancel",
8667                "attempt-cancel",
8668                "rust-worker",
8669                json!({"late":true}),
8670                JSON_CODEC,
8671            )
8672            .await
8673            .expect_err("late completion must be refused");
8674        assert!(activity_task_rejection_is_final(&error));
8675        let Error::ActivityTaskRejected(rejection) = error else {
8676            panic!("expected typed activity rejection");
8677        };
8678        assert_eq!(rejection.status, 409);
8679        assert_eq!(rejection.reason, "run_cancelled");
8680        assert!(rejection.cancel_requested);
8681        assert_eq!(rejection.can_continue, Some(false));
8682    }
8683
8684    #[tokio::test]
8685    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
8686        let server = MockWorkerServer::cancelled_activity();
8687        let client = Client::builder(server.base_url())
8688            .timeout(Duration::from_secs(2))
8689            .build()
8690            .expect("client");
8691        let cancellation_observed = Arc::new(AtomicBool::new(false));
8692        let observed = Arc::clone(&cancellation_observed);
8693        let mut worker = Worker::new(client.clone(), "rust-workers")
8694            .worker_id("rust-cancel-worker")
8695            .poll_timeout(Duration::from_millis(10));
8696        worker.register_activity("cancel-aware", move |ctx, _args| {
8697            let observed = Arc::clone(&observed);
8698            async move {
8699                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
8700                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
8701                Ok(json!({"late":"completion"}))
8702            }
8703        });
8704
8705        assert_eq!(
8706            worker.run_once().await.expect("cancelled attempt handled"),
8707            1
8708        );
8709        assert!(cancellation_observed.load(Ordering::SeqCst));
8710        assert_eq!(
8711            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
8712            1
8713        );
8714
8715        let mut restarted = Worker::new(client, "rust-workers")
8716            .worker_id("rust-cancel-worker-restarted")
8717            .poll_timeout(Duration::from_millis(10));
8718        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
8719        assert_eq!(
8720            restarted
8721                .run_once()
8722                .await
8723                .expect("replacement worker continues polling"),
8724            0
8725        );
8726    }
8727
8728    #[tokio::test]
8729    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
8730        let server = MockWorkerServer::start();
8731        let client = Client::builder(server.base_url())
8732            .timeout(Duration::from_secs(2))
8733            .build()
8734            .expect("client");
8735
8736        client
8737            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
8738            .await
8739            .expect("register");
8740        client
8741            .heartbeat_worker("capture-worker", 1, 1)
8742            .await
8743            .expect("heartbeat");
8744        client
8745            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
8746            .await
8747            .expect("workflow poll");
8748        client
8749            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
8750            .await
8751            .expect("activity poll");
8752
8753        for path in [
8754            "/api/worker/register",
8755            "/api/worker/heartbeat",
8756            "/api/worker/workflow-tasks/poll",
8757            "/api/worker/activity-tasks/poll",
8758        ] {
8759            assert_eq!(
8760                server.worker_protocol_for(path).as_deref(),
8761                Some(WORKER_PROTOCOL_VERSION),
8762                "unexpected protocol for {path}"
8763            );
8764        }
8765
8766        assert_eq!(
8767            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
8768            1
8769        );
8770        assert_eq!(
8771            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
8772            1
8773        );
8774    }
8775
8776    #[tokio::test]
8777    async fn query_task_endpoints_send_the_query_feature_protocol() {
8778        let server = MockWorkerServer::start();
8779        let client = Client::builder(server.base_url())
8780            .timeout(Duration::from_secs(2))
8781            .build()
8782            .expect("client");
8783
8784        client
8785            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8786            .await
8787            .expect("query poll");
8788        client
8789            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
8790            .await
8791            .expect("query complete");
8792        client
8793            .fail_query_task(
8794                "query-capture",
8795                "capture-worker",
8796                1,
8797                "failed",
8798                "query_rejected",
8799                "QueryFailed",
8800            )
8801            .await
8802            .expect("query fail");
8803
8804        for path in [
8805            "/api/worker/query-tasks/poll",
8806            "/api/worker/query-tasks/query-capture/complete",
8807            "/api/worker/query-tasks/query-capture/fail",
8808        ] {
8809            assert_eq!(
8810                server.worker_protocol_for(path).as_deref(),
8811                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
8812                "unexpected protocol for {path}"
8813            );
8814        }
8815
8816        assert_eq!(
8817            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
8818            1
8819        );
8820    }
8821
8822    #[tokio::test]
8823    async fn query_protocol_rejection_from_older_server_is_typed() {
8824        let server = MockWorkerServer::reject_query_protocol();
8825        let client = Client::builder(server.base_url())
8826            .timeout(Duration::from_secs(2))
8827            .build()
8828            .expect("client");
8829
8830        let error = client
8831            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8832            .await
8833            .expect_err("server below query protocol floor must reject");
8834        let Error::Protocol(failure) = error else {
8835            panic!("expected typed protocol failure");
8836        };
8837
8838        assert_eq!(failure.status, 400);
8839        assert_eq!(failure.reason, "unsupported_protocol_version");
8840        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
8841        assert_eq!(
8842            failure.requested_version.as_deref(),
8843            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8844        );
8845        assert_eq!(
8846            server
8847                .worker_protocol_for("/api/worker/query-tasks/poll")
8848                .as_deref(),
8849            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8850        );
8851    }
8852
8853    #[tokio::test]
8854    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
8855        let server = MockWorkerServer::reject_query_protocol();
8856        let client = Client::builder(server.base_url())
8857            .timeout(Duration::from_secs(2))
8858            .build()
8859            .expect("client");
8860        let mut worker = Worker::new(client, "rust-workers")
8861            .worker_id("baseline-worker")
8862            .poll_timeout(Duration::from_millis(10));
8863
8864        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
8865            Ok(Value::Null)
8866        });
8867
8868        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
8869        assert_eq!(
8870            server
8871                .worker_protocol_for("/api/worker/workflow-tasks/poll")
8872                .as_deref(),
8873            Some(WORKER_PROTOCOL_VERSION)
8874        );
8875        assert_eq!(
8876            server.worker_protocol_for("/api/worker/query-tasks/poll"),
8877            None,
8878            "a worker without query handlers must not use the query-task endpoint"
8879        );
8880    }
8881
8882    #[tokio::test]
8883    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
8884        let server = MockWorkerServer::reject_query_completion();
8885        let client = Client::builder(server.base_url())
8886            .timeout(Duration::from_secs(2))
8887            .build()
8888            .expect("client");
8889
8890        let error = client
8891            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
8892            .await
8893            .expect_err("expired completion must be rejected");
8894        let Error::QueryFailed(failure) = error else {
8895            panic!("expected typed query failure");
8896        };
8897        assert_eq!(failure.status, 409);
8898        assert_eq!(failure.reason, "query_task_timed_out");
8899
8900        let mut worker = Worker::new(client, "rust-workers")
8901            .worker_id("late-worker")
8902            .poll_timeout(Duration::from_millis(10));
8903        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8904        worker.register_query(
8905            "counter",
8906            "current",
8907            |_ctx, _args| async move { Ok(json!(8)) },
8908        );
8909
8910        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
8911        assert_eq!(
8912            worker
8913                .run_once()
8914                .await
8915                .expect("worker continues after late completion"),
8916            0
8917        );
8918        assert_eq!(
8919            server.request_count("/api/worker/query-tasks/query-late/complete"),
8920            2
8921        );
8922        assert_eq!(
8923            server.request_count("/api/worker/query-tasks/query-late/fail"),
8924            0,
8925            "a server completion rejection must not be reported as an encoding failure"
8926        );
8927    }
8928
8929    #[tokio::test]
8930    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
8931        let server = MockWorkerServer::start();
8932        let client = Client::builder(server.base_url())
8933            .timeout(Duration::from_secs(2))
8934            .build()
8935            .expect("client");
8936        let mut worker = Worker::new(client, "rust-workers")
8937            .worker_id("activity-only-worker")
8938            .poll_timeout(Duration::from_millis(10));
8939
8940        worker.register_activity(
8941            "activity.only",
8942            |_ctx, _args| async move { Ok(Value::Null) },
8943        );
8944
8945        worker.run_until(async {}).await.expect("run worker");
8946    }
8947
8948    #[tokio::test]
8949    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
8950        let server = MockWorkerServer::start();
8951        let client = Client::builder(server.base_url())
8952            .timeout(Duration::from_secs(2))
8953            .build()
8954            .expect("client");
8955        let mut worker = Worker::new(client, "rust-workers")
8956            .worker_id("workflow-only-worker")
8957            .poll_timeout(Duration::from_millis(10));
8958
8959        worker.register_workflow(
8960            "workflow.only",
8961            |_ctx, _input| async move { Ok(Value::Null) },
8962        );
8963
8964        worker.run_until(async {}).await.expect("run worker");
8965    }
8966
8967    #[tokio::test]
8968    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
8969        let server = MockWorkerServer::start();
8970        let client = Client::builder(server.base_url())
8971            .timeout(Duration::from_secs(2))
8972            .build()
8973            .expect("client");
8974        let observations = Arc::new(Mutex::new(Vec::new()));
8975        let observed = Arc::clone(&observations);
8976        let mut worker = Worker::new(client, "rust-workers")
8977            .worker_id("observed-heartbeat-worker")
8978            .poll_timeout(Duration::from_millis(10))
8979            .on_worker_heartbeat(move |observation| {
8980                observed
8981                    .lock()
8982                    .expect("heartbeat observations")
8983                    .push(observation.clone());
8984            });
8985
8986        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
8987            Ok(Value::Null)
8988        });
8989        worker
8990            .run_until(tokio::time::sleep(Duration::from_millis(20)))
8991            .await
8992            .expect("run worker");
8993
8994        let observations = observations.lock().expect("heartbeat observations");
8995        let first = observations.first().expect("heartbeat acknowledgement");
8996        assert_eq!(first.worker_id, "observed-heartbeat-worker");
8997        assert_eq!(first.task_queue, "rust-workers");
8998        assert!(first.acknowledged_at_unix_millis > 0);
8999        assert_eq!(first.acknowledgement, json!({}));
9000    }
9001
9002    #[tokio::test]
9003    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
9004        let server = MockWorkerServer::delayed_heartbeat_worker();
9005        let client = Client::builder(server.base_url())
9006            .timeout(Duration::from_secs(3))
9007            .build()
9008            .expect("client");
9009        let observations = Arc::new(Mutex::new(Vec::new()));
9010        let observed = Arc::clone(&observations);
9011        let mut worker = Worker::new(client, "rust-snapshot-workers")
9012            .worker_id("rust-snapshot-worker")
9013            .poll_timeout(Duration::from_millis(10))
9014            .on_worker_heartbeat(move |observation| {
9015                observed
9016                    .lock()
9017                    .expect("heartbeat observations")
9018                    .push(observation.clone());
9019            });
9020
9021        worker.register_workflow("snapshot", |ctx, _input| async move {
9022            ctx.wait_signal("finish").await?;
9023            Ok(json!({"status": "finished"}))
9024        });
9025        worker.register_query("snapshot", "current", |ctx, _args| async move {
9026            Ok(json!(ctx
9027                .signals("increment")
9028                .iter()
9029                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
9030                .sum::<i64>()))
9031        });
9032        worker.register_activity("cancel-aware", |_ctx, _args| async move {
9033            Ok(json!({"late": "completion"}))
9034        });
9035
9036        worker
9037            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
9038            .await
9039            .expect("delayed heartbeat must allow a clean worker shutdown");
9040
9041        let observations = observations.lock().expect("heartbeat observations");
9042        assert!(
9043            observations.len() >= 3,
9044            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
9045        );
9046        assert!(
9047            observations.windows(2).all(|pair| {
9048                pair[1].acknowledged_at_unix_millis
9049                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
9050                    >= 850
9051            }),
9052            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
9053        );
9054        drop(observations);
9055
9056        let heartbeat_times = server.request_times("/api/worker/heartbeat");
9057        let delayed_request_at = *heartbeat_times
9058            .get(1)
9059            .expect("intentionally delayed heartbeat request");
9060        let delay_window_start = delayed_request_at + Duration::from_millis(100);
9061        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
9062        for path in [
9063            "/api/worker/workflow-tasks/poll",
9064            "/api/worker/activity-tasks/poll",
9065            "/api/worker/query-tasks/poll",
9066        ] {
9067            assert!(
9068                server
9069                    .request_times(path)
9070                    .iter()
9071                    .any(|received_at| *received_at >= delay_window_start
9072                        && *received_at <= delay_window_end),
9073                "{path} must keep polling while a heartbeat acknowledgement is delayed"
9074            );
9075        }
9076        assert!(
9077            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
9078            "workflow work must be settled"
9079        );
9080        assert!(
9081            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
9082            "activity work must be settled"
9083        );
9084        assert!(
9085            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
9086            "query work must be settled"
9087        );
9088    }
9089
9090    #[tokio::test]
9091    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
9092        let server = MockWorkerServer::heartbeat_retry_worker();
9093        let client = Client::builder(server.base_url())
9094            .timeout(Duration::from_secs(2))
9095            .build()
9096            .expect("client");
9097        let observations = Arc::new(Mutex::new(Vec::new()));
9098        let observed = Arc::clone(&observations);
9099        let worker = Worker::new(client, "rust-workers")
9100            .worker_id("heartbeat-retry-worker")
9101            .retry_policy(WorkerRetryPolicy {
9102                max_retries: 1,
9103                initial_backoff: Duration::from_millis(300),
9104                max_backoff: Duration::from_millis(300),
9105            })
9106            .on_worker_heartbeat(move |observation| {
9107                observed
9108                    .lock()
9109                    .expect("heartbeat observations")
9110                    .push(observation.clone());
9111            });
9112
9113        worker
9114            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
9115            .await
9116            .expect("retryable heartbeat failure must remain bounded and recover");
9117
9118        let observations = observations.lock().expect("heartbeat observations");
9119        assert!(observations.len() >= 3, "heartbeat retry must recover");
9120        assert!(
9121            observations.windows(2).all(|pair| {
9122                pair[1]
9123                    .acknowledged_at_unix_millis
9124                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
9125                    >= 850
9126            }),
9127            "a successful retry must start a fresh advertised cadence: {observations:?}"
9128        );
9129        assert_eq!(
9130            server.request_count("/api/worker/heartbeat"),
9131            observations.len() + 1,
9132            "one retryable failure must add exactly one bounded request"
9133        );
9134    }
9135
9136    #[tokio::test]
9137    async fn query_enabled_worker_stays_live_when_signal_replay_emits_no_commands() {
9138        let server = MockWorkerServer::waiting_query_worker();
9139        let client = Client::builder(server.base_url())
9140            .timeout(Duration::from_secs(2))
9141            .build()
9142            .expect("client");
9143        let observations = Arc::new(Mutex::new(Vec::new()));
9144        let observed = Arc::clone(&observations);
9145        let mut worker = Worker::new(client, "rust-snapshot-workers")
9146            .worker_id("rust-snapshot-worker")
9147            .poll_timeout(Duration::from_millis(10))
9148            .on_worker_heartbeat(move |observation| {
9149                observed
9150                    .lock()
9151                    .expect("heartbeat observations")
9152                    .push(observation.clone());
9153            });
9154
9155        worker.register_workflow("snapshot", |ctx, _input| async move {
9156            ctx.wait_signal("finish").await?;
9157            Ok(json!({"status": "finished"}))
9158        });
9159        worker.register_query("snapshot", "current", |ctx, _args| async move {
9160            let current = ctx
9161                .signals("increment")
9162                .iter()
9163                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
9164                .sum::<i64>();
9165            Ok(json!(current))
9166        });
9167
9168        worker
9169            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
9170            .await
9171            .expect("pending workflow and query poller must remain live until shutdown");
9172
9173        assert!(
9174            observations.lock().expect("heartbeat observations").len() >= 4,
9175            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
9176        );
9177        assert!(
9178            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
9179            "workflow polling must continue after empty replay acknowledgements"
9180        );
9181        assert!(
9182            server.request_count("/api/worker/query-tasks/poll") >= 2,
9183            "query polling must continue after serving the current query"
9184        );
9185        assert_eq!(
9186            server.request_body("/api/worker/register")["capabilities"],
9187            json!([QUERY_TASKS_CAPABILITY])
9188        );
9189
9190        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
9191            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
9192            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
9193            let failure = server.request_body(&fail_path);
9194            assert_eq!(
9195                failure["failure"]["type"],
9196                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
9197            );
9198            assert_eq!(server.request_count(&completion_path), 0);
9199        }
9200
9201        let query_completion =
9202            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
9203        assert_eq!(query_completion["result"], json!(8));
9204    }
9205
9206    #[tokio::test]
9207    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
9208        let server = MockWorkerServer::transient_worker_failures();
9209        let client = Client::builder(server.base_url())
9210            .timeout(Duration::from_secs(2))
9211            .build()
9212            .expect("client");
9213        let mut worker = Worker::new(client, "rust-workers")
9214            .worker_id("retry-worker")
9215            .poll_timeout(Duration::from_millis(10))
9216            .retry_policy(WorkerRetryPolicy {
9217                max_retries: 2,
9218                initial_backoff: Duration::from_millis(1),
9219                max_backoff: Duration::from_millis(1),
9220            });
9221        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
9222        worker.register_activity(
9223            "counter.activity",
9224            |_ctx, _input| async move { Ok(Value::Null) },
9225        );
9226        worker.register_query(
9227            "counter",
9228            "current",
9229            |_ctx, _args| async move { Ok(json!(8)) },
9230        );
9231
9232        worker
9233            .run_until(tokio::time::sleep(Duration::from_millis(75)))
9234            .await
9235            .expect("transient failures must not stop the worker");
9236
9237        for path in [
9238            "/api/worker/heartbeat",
9239            "/api/worker/workflow-tasks/poll",
9240            "/api/worker/activity-tasks/poll",
9241            "/api/worker/query-tasks/poll",
9242        ] {
9243            assert!(
9244                server.request_count(path) >= 2,
9245                "{path} must continue after its transient failure"
9246            );
9247        }
9248    }
9249
9250    #[tokio::test]
9251    async fn worker_bounds_transport_retries() {
9252        let server = MockWorkerServer::unavailable_polls();
9253        let client = Client::builder(server.base_url())
9254            .timeout(Duration::from_secs(2))
9255            .build()
9256            .expect("client");
9257        let mut worker = Worker::new(client, "rust-workers")
9258            .worker_id("bounded-retry-worker")
9259            .poll_timeout(Duration::from_millis(10))
9260            .retry_policy(WorkerRetryPolicy {
9261                max_retries: 2,
9262                initial_backoff: Duration::from_millis(1),
9263                max_backoff: Duration::from_millis(1),
9264            });
9265        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
9266
9267        let error = worker.run().await.expect_err("retry bound must terminate");
9268        assert!(matches!(error, Error::Transport(_)));
9269        assert_eq!(
9270            server.request_count("/api/worker/workflow-tasks/poll"),
9271            3,
9272            "one initial request plus two retries"
9273        );
9274    }
9275
9276    #[tokio::test]
9277    async fn worker_does_not_retry_authentication_failures() {
9278        let server = MockWorkerServer::unauthorized_polls();
9279        let client = Client::builder(server.base_url())
9280            .timeout(Duration::from_secs(2))
9281            .build()
9282            .expect("client");
9283        let mut worker = Worker::new(client, "rust-workers")
9284            .worker_id("unauthorized-worker")
9285            .poll_timeout(Duration::from_millis(10));
9286        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
9287
9288        let error = worker
9289            .run()
9290            .await
9291            .expect_err("authentication must terminate");
9292        let Error::Http { status, body } = error else {
9293            panic!("expected stable HTTP authentication error");
9294        };
9295        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
9296        assert!(body.contains("authentication_failed"));
9297        assert_eq!(
9298            server.request_count("/api/worker/workflow-tasks/poll"),
9299            1,
9300            "authentication failures must not be retried"
9301        );
9302    }
9303
9304    #[derive(Clone, Debug)]
9305    struct CapturedRequest {
9306        path: String,
9307        worker_protocol: Option<String>,
9308        body: String,
9309        received_at: Instant,
9310    }
9311
9312    struct MockWorkerServer {
9313        addr: SocketAddr,
9314        stop: Arc<AtomicBool>,
9315        requests: Arc<Mutex<Vec<CapturedRequest>>>,
9316        thread: Option<thread::JoinHandle<()>>,
9317    }
9318
9319    #[derive(Clone, Copy, Default)]
9320    struct MockWorkerBehavior {
9321        reject_query_protocol: bool,
9322        reject_query_completion: bool,
9323        waiting_query_worker: bool,
9324        poll_failures_per_path: usize,
9325        heartbeat_failures: usize,
9326        heartbeat_failure_request: Option<usize>,
9327        delayed_heartbeat_request: Option<usize>,
9328        heartbeat_response_delay: Duration,
9329        concurrent_requests: bool,
9330        unauthorized_polls: bool,
9331        cancelled_activity: bool,
9332        draining_polls: bool,
9333    }
9334
9335    impl MockWorkerServer {
9336        fn start() -> Self {
9337            Self::start_with_behavior(MockWorkerBehavior::default())
9338        }
9339
9340        fn reject_query_protocol() -> Self {
9341            Self::start_with_behavior(MockWorkerBehavior {
9342                reject_query_protocol: true,
9343                ..MockWorkerBehavior::default()
9344            })
9345        }
9346
9347        fn reject_query_completion() -> Self {
9348            Self::start_with_behavior(MockWorkerBehavior {
9349                reject_query_completion: true,
9350                ..MockWorkerBehavior::default()
9351            })
9352        }
9353
9354        fn waiting_query_worker() -> Self {
9355            Self::start_with_behavior(MockWorkerBehavior {
9356                waiting_query_worker: true,
9357                ..MockWorkerBehavior::default()
9358            })
9359        }
9360
9361        fn transient_worker_failures() -> Self {
9362            Self::start_with_behavior(MockWorkerBehavior {
9363                poll_failures_per_path: 1,
9364                heartbeat_failures: 1,
9365                ..MockWorkerBehavior::default()
9366            })
9367        }
9368
9369        fn delayed_heartbeat_worker() -> Self {
9370            Self::start_with_behavior(MockWorkerBehavior {
9371                waiting_query_worker: true,
9372                delayed_heartbeat_request: Some(2),
9373                heartbeat_response_delay: Duration::from_millis(1_500),
9374                concurrent_requests: true,
9375                cancelled_activity: true,
9376                ..MockWorkerBehavior::default()
9377            })
9378        }
9379
9380        fn heartbeat_retry_worker() -> Self {
9381            Self::start_with_behavior(MockWorkerBehavior {
9382                waiting_query_worker: true,
9383                heartbeat_failure_request: Some(2),
9384                concurrent_requests: true,
9385                ..MockWorkerBehavior::default()
9386            })
9387        }
9388
9389        fn unavailable_polls() -> Self {
9390            Self::start_with_behavior(MockWorkerBehavior {
9391                poll_failures_per_path: usize::MAX,
9392                ..MockWorkerBehavior::default()
9393            })
9394        }
9395
9396        fn unauthorized_polls() -> Self {
9397            Self::start_with_behavior(MockWorkerBehavior {
9398                unauthorized_polls: true,
9399                ..MockWorkerBehavior::default()
9400            })
9401        }
9402
9403        fn cancelled_activity() -> Self {
9404            Self::start_with_behavior(MockWorkerBehavior {
9405                cancelled_activity: true,
9406                ..MockWorkerBehavior::default()
9407            })
9408        }
9409
9410        fn draining_polls() -> Self {
9411            Self::start_with_behavior(MockWorkerBehavior {
9412                draining_polls: true,
9413                ..MockWorkerBehavior::default()
9414            })
9415        }
9416
9417        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
9418            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
9419            listener
9420                .set_nonblocking(true)
9421                .expect("configure mock listener");
9422            let addr = listener.local_addr().expect("mock server address");
9423            let stop = Arc::new(AtomicBool::new(false));
9424            let server_stop = Arc::clone(&stop);
9425            let requests = Arc::new(Mutex::new(Vec::new()));
9426            let server_requests = Arc::clone(&requests);
9427            let thread = thread::spawn(move || {
9428                let mut request_threads = Vec::new();
9429                while !server_stop.load(Ordering::SeqCst) {
9430                    match listener.accept() {
9431                        Ok((mut stream, _)) => {
9432                            if behavior.concurrent_requests {
9433                                let requests = Arc::clone(&server_requests);
9434                                request_threads.push(thread::spawn(move || {
9435                                    handle_mock_worker_request(&mut stream, &requests, behavior)
9436                                }));
9437                            } else {
9438                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
9439                            }
9440                        }
9441                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
9442                            let mut index = 0;
9443                            while index < request_threads.len() {
9444                                if request_threads[index].is_finished() {
9445                                    request_threads
9446                                        .swap_remove(index)
9447                                        .join()
9448                                        .expect("join mock request");
9449                                } else {
9450                                    index += 1;
9451                                }
9452                            }
9453                            thread::sleep(Duration::from_millis(5));
9454                        }
9455                        Err(_) => break,
9456                    }
9457                }
9458                for request_thread in request_threads {
9459                    request_thread.join().expect("join mock request");
9460                }
9461            });
9462
9463            Self {
9464                addr,
9465                stop,
9466                requests,
9467                thread: Some(thread),
9468            }
9469        }
9470
9471        fn base_url(&self) -> String {
9472            format!("http://{}", self.addr)
9473        }
9474
9475        fn worker_protocol_for(&self, path: &str) -> Option<String> {
9476            self.requests
9477                .lock()
9478                .expect("captured requests")
9479                .iter()
9480                .find(|request| request.path == path)
9481                .and_then(|request| request.worker_protocol.clone())
9482        }
9483
9484        fn request_count(&self, path: &str) -> usize {
9485            self.requests
9486                .lock()
9487                .expect("captured requests")
9488                .iter()
9489                .filter(|request| request.path == path)
9490                .count()
9491        }
9492
9493        fn request_times(&self, path: &str) -> Vec<Instant> {
9494            self.requests
9495                .lock()
9496                .expect("captured requests")
9497                .iter()
9498                .filter(|request| request.path == path)
9499                .map(|request| request.received_at)
9500                .collect()
9501        }
9502
9503        fn request_body(&self, path: &str) -> Value {
9504            let requests = self.requests.lock().expect("captured requests");
9505            let body = &requests
9506                .iter()
9507                .find(|request| request.path == path)
9508                .unwrap_or_else(|| panic!("missing request for {path}"))
9509                .body;
9510            serde_json::from_str(body).unwrap_or_else(|error| {
9511                panic!("invalid JSON request body for {path}: {error}: {body:?}")
9512            })
9513        }
9514    }
9515
9516    impl Drop for MockWorkerServer {
9517        fn drop(&mut self) {
9518            self.stop.store(true, Ordering::SeqCst);
9519            let _ = TcpStream::connect(self.addr);
9520
9521            if let Some(thread) = self.thread.take() {
9522                thread.join().expect("join mock server");
9523            }
9524        }
9525    }
9526
9527    fn handle_mock_worker_request(
9528        stream: &mut TcpStream,
9529        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
9530        behavior: MockWorkerBehavior,
9531    ) {
9532        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
9533        let mut buffer = [0_u8; 8192];
9534        let mut request = Vec::new();
9535
9536        loop {
9537            match stream.read(&mut buffer) {
9538                Ok(0) => break,
9539                Ok(read) => {
9540                    request.extend_from_slice(&buffer[..read]);
9541                    if mock_request_is_complete(&request) {
9542                        break;
9543                    }
9544                }
9545                Err(error)
9546                    if matches!(
9547                        error.kind(),
9548                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
9549                    ) =>
9550                {
9551                    break;
9552                }
9553                Err(_) => return,
9554            }
9555        }
9556
9557        let request = String::from_utf8_lossy(&request);
9558        let body = request
9559            .split_once("\r\n\r\n")
9560            .map(|(_, body)| body)
9561            .unwrap_or_default();
9562        let path = request
9563            .lines()
9564            .next()
9565            .and_then(|line| line.split_whitespace().nth(1))
9566            .unwrap_or_default();
9567        let worker_protocol = request.lines().find_map(|line| {
9568            let (name, value) = line.split_once(':')?;
9569            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
9570                .then(|| value.trim().to_string())
9571        });
9572        let request_number = {
9573            let mut requests = requests.lock().expect("captured requests");
9574            requests.push(CapturedRequest {
9575                path: path.to_string(),
9576                worker_protocol: worker_protocol.clone(),
9577                body: body.to_string(),
9578                received_at: Instant::now(),
9579            });
9580            requests
9581                .iter()
9582                .filter(|request| request.path == path)
9583                .count()
9584        };
9585
9586        let is_poll = matches!(
9587            path,
9588            "/api/worker/workflow-tasks/poll"
9589                | "/api/worker/activity-tasks/poll"
9590                | "/api/worker/query-tasks/poll"
9591        );
9592        if is_poll && request_number <= behavior.poll_failures_per_path {
9593            return;
9594        }
9595        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
9596            return;
9597        }
9598        if path == "/api/worker/heartbeat"
9599            && behavior.heartbeat_failure_request == Some(request_number)
9600        {
9601            return;
9602        }
9603        if path == "/api/worker/heartbeat"
9604            && behavior.delayed_heartbeat_request == Some(request_number)
9605        {
9606            thread::sleep(behavior.heartbeat_response_delay);
9607        }
9608        if behavior.unauthorized_polls && is_poll {
9609            write_mock_response(
9610                stream,
9611                "401 Unauthorized",
9612                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
9613            );
9614            return;
9615        }
9616        if behavior.draining_polls && is_poll {
9617            write_mock_response(
9618                stream,
9619                "409 Conflict",
9620                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
9621            );
9622            return;
9623        }
9624
9625        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
9626            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
9627            let body = format!(
9628                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
9629            );
9630            write_mock_response(stream, "400 Bad Request", &body);
9631            return;
9632        }
9633
9634        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
9635        {
9636            write_mock_response(
9637                stream,
9638                "409 Conflict",
9639                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
9640            );
9641            return;
9642        }
9643
9644        if behavior.waiting_query_worker {
9645            if path == "/api/worker/workflow-tasks/poll" && request_number <= 2 {
9646                let amounts = if request_number == 1 {
9647                    vec![3]
9648                } else {
9649                    vec![3, 5]
9650                };
9651                let task_id = if request_number == 1 {
9652                    "snapshot-wait-3"
9653                } else {
9654                    "snapshot-wait-5"
9655                };
9656                let history_events = std::iter::once(json!({
9657                    "event_type": "SignalWaitOpened",
9658                    "payload": {"sequence": 1, "signal_name": "finish"}
9659                }))
9660                .chain(amounts.iter().enumerate().map(|(index, amount)| {
9661                    json!({
9662                        "event_type": "SignalReceived",
9663                        "payload": {
9664                            "signal_id": format!("increment-{amount}"),
9665                            "signal_name": "increment",
9666                            "workflow_sequence": index + 2,
9667                            "payload_codec": DEFAULT_CODEC,
9668                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9669                                .expect("Avro signal envelope")
9670                        }
9671                    })
9672                }))
9673                .collect::<Vec<_>>();
9674                let body = json!({
9675                    "task": {
9676                        "task_id": task_id,
9677                        "workflow_id": "snapshot-1",
9678                        "run_id": "snapshot-run-1",
9679                        "workflow_type": "snapshot",
9680                        "payload_codec": DEFAULT_CODEC,
9681                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9682                            .expect("Avro workflow arguments"),
9683                        "history_events": history_events,
9684                        "workflow_task_attempt": 1,
9685                        "workflow_signal_id": format!("increment-{}", amounts.last().expect("amount")),
9686                        "signal_name": "increment",
9687                        "signal_arguments": encode_value_envelope(
9688                            &json!([amounts.last().expect("amount")]),
9689                            DEFAULT_CODEC,
9690                        )
9691                        .expect("Avro resume signal"),
9692                        "lease_owner": "rust-snapshot-worker"
9693                    }
9694                })
9695                .to_string();
9696                write_mock_response(stream, "200 OK", &body);
9697                return;
9698            }
9699
9700            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
9701                let history_events = [3, 5]
9702                    .into_iter()
9703                    .enumerate()
9704                    .map(|(index, amount)| {
9705                        json!({
9706                            "event_type": "SignalReceived",
9707                            "payload": {
9708                                "signal_id": format!("increment-{amount}"),
9709                                "signal_name": "increment",
9710                                "workflow_sequence": index + 2,
9711                                "payload_codec": DEFAULT_CODEC,
9712                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9713                                    .expect("Avro query signal envelope")
9714                            }
9715                        })
9716                    })
9717                    .collect::<Vec<_>>();
9718                let body = json!({
9719                    "task": {
9720                        "query_task_id": "snapshot-current",
9721                        "query_task_attempt": 1,
9722                        "lease_owner": "rust-snapshot-worker",
9723                        "workflow_id": "snapshot-1",
9724                        "run_id": "snapshot-run-1",
9725                        "workflow_type": "snapshot",
9726                        "query_name": "current",
9727                        "payload_codec": DEFAULT_CODEC,
9728                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9729                            .expect("Avro workflow arguments"),
9730                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9731                            .expect("Avro query arguments"),
9732                        "history_events": history_events,
9733                        "run_status": "waiting"
9734                    }
9735                })
9736                .to_string();
9737                write_mock_response(stream, "200 OK", &body);
9738                return;
9739            }
9740
9741            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
9742                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
9743            {
9744                write_mock_response(
9745                    stream,
9746                    "200 OK",
9747                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
9748                );
9749                return;
9750            }
9751
9752            if path == "/api/worker/query-tasks/snapshot-current/complete" {
9753                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
9754                return;
9755            }
9756        }
9757
9758        let (status, body) = match path {
9759            "/api/workflows" => (
9760                "201 Created",
9761                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
9762            ),
9763            "/api/worker/register" if behavior.waiting_query_worker => (
9764                "200 OK",
9765                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
9766            ),
9767            "/api/worker/register" => (
9768                "200 OK",
9769                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
9770            ),
9771            "/api/worker/heartbeat" => ("200 OK", "{}"),
9772            "/api/worker/activity-tasks/poll"
9773                if behavior.cancelled_activity && request_number == 1 =>
9774            {
9775                (
9776                    "200 OK",
9777                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
9778                )
9779            }
9780            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
9781                ("200 OK", r#"{"task":null}"#)
9782            }
9783            "/api/worker/query-tasks/poll"
9784                if behavior.reject_query_completion && request_number == 1 =>
9785            {
9786                (
9787                    "200 OK",
9788                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
9789                )
9790            }
9791            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
9792            "/api/worker/query-tasks/query-capture/complete"
9793            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
9794            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
9795                "200 OK",
9796                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
9797            ),
9798            "/api/worker/activity-tasks/activity-cancel/complete" => (
9799                "409 Conflict",
9800                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
9801            ),
9802            "/api/workflows/counter-1/query/current" => (
9803                "200 OK",
9804                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
9805            ),
9806            "/api/workflows/counter-1/query/missing" => (
9807                "404 Not Found",
9808                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
9809            ),
9810            "/api/workflows/wf-lifecycle/cancel" => (
9811                "200 OK",
9812                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
9813            ),
9814            "/api/workflows/wf-lifecycle/terminate" => (
9815                "200 OK",
9816                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
9817            ),
9818            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
9819                "200 OK",
9820                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
9821            ),
9822            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
9823                "200 OK",
9824                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
9825            ),
9826            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
9827            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
9828                "409 Conflict",
9829                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
9830            ),
9831            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
9832                "200 OK",
9833                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
9834            ),
9835            "/api/workflows/wf-cancelled" => (
9836                "200 OK",
9837                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
9838            ),
9839            "/api/workflows/wf-terminated" => (
9840                "200 OK",
9841                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
9842            ),
9843            "/api/workflows/wf-timed-out" => (
9844                "200 OK",
9845                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
9846            ),
9847            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
9848                "200 OK",
9849                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
9850            ),
9851            "/api/workflows/wf-selected" => (
9852                "200 OK",
9853                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
9854            ),
9855            "/api/workflows/wf-selected/runs/run-selected" => (
9856                "200 OK",
9857                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
9858            ),
9859            _ => ("404 Not Found", r#"{"message":"not found"}"#),
9860        };
9861        write_mock_response(stream, status, body);
9862    }
9863
9864    fn mock_request_is_complete(request: &[u8]) -> bool {
9865        let Some(header_end) = request
9866            .windows(4)
9867            .position(|window| window == b"\r\n\r\n")
9868            .map(|position| position + 4)
9869        else {
9870            return false;
9871        };
9872        let headers = String::from_utf8_lossy(&request[..header_end]);
9873        let content_length = headers.lines().find_map(|line| {
9874            let (name, value) = line.split_once(':')?;
9875            name.eq_ignore_ascii_case("content-length")
9876                .then(|| value.trim().parse::<usize>().ok())
9877                .flatten()
9878        });
9879
9880        request.len() >= header_end + content_length.unwrap_or(0)
9881    }
9882
9883    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
9884        let response = format!(
9885            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
9886            body.len()
9887        );
9888
9889        let _ = stream.write_all(response.as_bytes());
9890        let _ = stream.flush();
9891    }
9892}