Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    pin::Pin,
8    sync::{
9        atomic::{AtomicBool, Ordering},
10        Arc, Mutex, OnceLock,
11    },
12    task::{Context as TaskContext, Poll},
13    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
14};
15
16use apache_avro::{from_avro_datum, from_value, to_avro_datum, to_value, Schema};
17use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
18use futures_util::{future::OptionFuture, task::noop_waker_ref};
19use serde::{de::DeserializeOwned, Deserialize, Serialize};
20pub use serde_json::{json, Value};
21use thiserror::Error;
22
23pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
24pub const CONTROL_PLANE_VERSION: &str = "2";
25pub const DEFAULT_CODEC: &str = "avro";
26pub const JSON_CODEC: &str = "json";
27pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
28/// Worker-registration capability for server-routed read-only queries.
29pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
30/// First additive worker protocol that defines query-task transport.
31pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
32
33const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
34const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
35    "Workflow task waiting for scheduled history.";
36const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
37
38const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
39    "lease_expired",
40    "query_task_not_found",
41    "query_task_not_leased",
42    "query_task_timed_out",
43];
44
45const AVRO_PAYLOAD_SCHEMA_JSON: &str = r#"{"type":"record","name":"Payload","namespace":"durable_workflow","fields":[{"name":"json","type":"string"},{"name":"version","type":"int","default":1}]}"#;
46const AVRO_PAYLOAD_VERSION: i32 = 1;
47
48static AVRO_PAYLOAD_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
49
50#[derive(Clone, Copy)]
51enum RequestProtocol {
52    ControlPlane,
53    Worker(&'static str),
54}
55
56impl RequestProtocol {
57    fn is_worker(self) -> bool {
58        matches!(self, Self::Worker(_))
59    }
60}
61
62pub type Result<T> = std::result::Result<T, Error>;
63
64#[derive(Debug, Error)]
65pub enum Error {
66    #[error("transport error: {0}")]
67    Transport(#[from] reqwest::Error),
68    #[error("json error: {0}")]
69    Json(#[from] serde_json::Error),
70    #[error("http {status}: {body}")]
71    Http {
72        status: reqwest::StatusCode,
73        body: String,
74    },
75    #[error("codec error: {0}")]
76    Codec(String),
77    #[error(transparent)]
78    QueryFailed(QueryFailure),
79    #[error(transparent)]
80    Protocol(ProtocolFailure),
81    #[error(transparent)]
82    NonDeterministicReplay(ReplayFailure),
83    #[error(transparent)]
84    ChildWorkflowFailed(ChildWorkflowFailure),
85    #[error(transparent)]
86    ActivityFailed(ActivityFailure),
87    #[error(transparent)]
88    WorkflowCommandRejected(WorkflowCommandRejection),
89    #[error(transparent)]
90    WorkflowFailed(WorkflowTerminalOutcome),
91    #[error(transparent)]
92    WorkflowCancelled(WorkflowTerminalOutcome),
93    #[error(transparent)]
94    WorkflowTerminated(WorkflowTerminalOutcome),
95    #[error(transparent)]
96    WorkflowTimedOut(WorkflowTerminalOutcome),
97    #[error(transparent)]
98    ActivityTaskRejected(ActivityTaskRejection),
99    #[error("workflow handler {0:?} is not registered")]
100    WorkflowNotRegistered(String),
101    #[error("activity handler {0:?} is not registered")]
102    ActivityNotRegistered(String),
103    #[error("workflow future yielded without emitting a durable command")]
104    WorkflowYieldedWithoutCommand,
105    #[error("workflow state lock is poisoned")]
106    WorkflowStatePoisoned,
107    #[error("timer duration is too large for the worker protocol")]
108    TimerDurationOverflow,
109    #[error("operation timed out")]
110    Timeout,
111    #[error("worker loop error: {0}")]
112    WorkerLoop(String),
113    #[error("invalid child workflow options: {0}")]
114    InvalidChildWorkflowOptions(String),
115    #[error(transparent)]
116    InvalidActivityOptions(ActivityOptionsError),
117}
118
119/// The lifecycle command sent to a workflow execution.
120#[derive(Clone, Copy, Debug, PartialEq, Eq)]
121pub enum WorkflowCommandKind {
122    Cancel,
123    Terminate,
124}
125
126impl WorkflowCommandKind {
127    fn as_str(self) -> &'static str {
128        match self {
129            Self::Cancel => "cancel",
130            Self::Terminate => "terminate",
131        }
132    }
133}
134
135/// Optional structured fields for a cancellation or termination request.
136#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
137pub struct WorkflowCommandOptions {
138    #[serde(skip_serializing_if = "Option::is_none")]
139    pub reason: Option<String>,
140    #[serde(skip_serializing_if = "Option::is_none")]
141    pub request_id: Option<String>,
142}
143
144/// Server-enforced timeout policy for a workflow start.
145///
146/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
147/// only bounds how long the caller waits. A server deadline produces a terminal
148/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
149/// `run_timeout`.
150#[derive(Clone, Debug, PartialEq, Eq)]
151pub struct WorkflowStartOptions {
152    pub execution_timeout_seconds: u64,
153    pub run_timeout_seconds: u64,
154}
155
156impl Default for WorkflowStartOptions {
157    fn default() -> Self {
158        Self {
159            execution_timeout_seconds: 3600,
160            run_timeout_seconds: 600,
161        }
162    }
163}
164
165impl WorkflowStartOptions {
166    pub fn new() -> Self {
167        Self::default()
168    }
169
170    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
171        self.execution_timeout_seconds = seconds;
172        self
173    }
174
175    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
176        self.run_timeout_seconds = seconds;
177        self
178    }
179
180    fn validate(&self) -> Result<()> {
181        if self.execution_timeout_seconds == 0 {
182            return Err(Error::Codec(
183                "execution_timeout_seconds must be at least 1".to_string(),
184            ));
185        }
186        if self.run_timeout_seconds == 0 {
187            return Err(Error::Codec(
188                "run_timeout_seconds must be at least 1".to_string(),
189            ));
190        }
191        if self.run_timeout_seconds > self.execution_timeout_seconds {
192            return Err(Error::Codec(
193                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
194            ));
195        }
196
197        Ok(())
198    }
199}
200
201impl WorkflowCommandOptions {
202    pub fn new() -> Self {
203        Self::default()
204    }
205
206    pub fn reason(mut self, reason: impl Into<String>) -> Self {
207        self.reason = Some(reason.into());
208        self
209    }
210
211    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
212        self.request_id = Some(request_id.into());
213        self
214    }
215}
216
217/// The accepted, machine-readable result of a lifecycle command.
218#[derive(Clone, Debug, PartialEq)]
219pub struct WorkflowCommandResult {
220    pub command: WorkflowCommandKind,
221    pub workflow_id: String,
222    pub run_id: Option<String>,
223    pub outcome: Option<String>,
224    pub reason: Option<String>,
225    pub command_status: Option<String>,
226    pub raw: Value,
227}
228
229/// A stable rejection returned by instance- or selected-run lifecycle commands.
230#[derive(Clone, Debug, Error)]
231#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
232pub struct WorkflowCommandRejection {
233    pub command: WorkflowCommandKind,
234    pub status: u16,
235    pub reason: String,
236    pub message: String,
237    pub workflow_id: String,
238    pub run_id: Option<String>,
239    pub target_scope: Option<String>,
240    pub body: Value,
241}
242
243/// Stable terminal categories returned by [`WorkflowHandle::result`].
244#[derive(Clone, Copy, Debug, PartialEq, Eq)]
245pub enum WorkflowTerminalKind {
246    Failed,
247    Cancelled,
248    Terminated,
249    TimedOut,
250}
251
252/// A typed terminal workflow outcome with durable identity and failure metadata.
253///
254/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
255/// of parsing its display representation. Fields remain `None` when an older
256/// server did not publish that metadata.
257#[derive(Clone, Debug, Error)]
258#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
259pub struct WorkflowTerminalOutcome {
260    pub kind: WorkflowTerminalKind,
261    pub workflow_id: String,
262    pub run_id: Option<String>,
263    pub reason: String,
264    pub failure_category: Option<String>,
265    pub failure_id: Option<String>,
266    pub exception_type: Option<String>,
267    pub exception_class: Option<String>,
268    pub non_retryable: Option<bool>,
269    pub message: Option<String>,
270    pub exception: Option<Value>,
271    pub raw: Value,
272}
273
274/// A worker-side activity settlement or heartbeat rejected by durable state.
275#[derive(Clone, Debug, Error)]
276#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
277pub struct ActivityTaskRejection {
278    pub operation: String,
279    pub status: u16,
280    pub reason: String,
281    pub task_id: String,
282    pub activity_attempt_id: String,
283    pub cancel_requested: bool,
284    pub can_continue: Option<bool>,
285    pub run_closed_reason: Option<String>,
286    pub body: Value,
287}
288
289/// Stable validation categories for [`ActivityOptions`].
290#[derive(Clone, Copy, Debug, PartialEq, Eq)]
291pub enum ActivityOptionsErrorKind {
292    EmptyTaskQueue,
293    EmptyRetryPolicy,
294    InvalidMaxAttempts,
295    BackoffWithoutRetryBudget,
296    TooManyBackoffIntervals,
297    InvalidBackoffCoefficient,
298    BackoffGenerationTooLarge,
299    BackoffOverflow,
300    EmptyNonRetryableErrorType,
301    TimeoutNotPositive,
302    TimeoutOverflow,
303    TimeoutOrder,
304}
305
306/// A machine-readable activity-options validation failure.
307#[derive(Clone, Debug, Error, PartialEq, Eq)]
308#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
309pub struct ActivityOptionsError {
310    pub kind: ActivityOptionsErrorKind,
311    pub field: Option<&'static str>,
312    pub message: String,
313}
314
315impl ActivityOptionsError {
316    fn new(
317        kind: ActivityOptionsErrorKind,
318        field: Option<&'static str>,
319        message: impl Into<String>,
320    ) -> Self {
321        Self {
322            kind,
323            field,
324            message: message.into(),
325        }
326    }
327}
328
329/// Stable terminal categories returned when an awaited activity does not succeed.
330#[derive(Clone, Copy, Debug, PartialEq, Eq)]
331pub enum ActivityFailureKind {
332    Failed,
333    Cancelled,
334    TimedOut,
335}
336
337/// A stable, machine-readable terminal activity failure.
338///
339/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
340/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
341#[derive(Clone, Debug, Error)]
342#[error("activity failed ({reason}): {message}")]
343pub struct ActivityFailure {
344    pub kind: ActivityFailureKind,
345    pub reason: String,
346    pub message: String,
347    pub activity_execution_id: Option<String>,
348    pub activity_attempt_id: Option<String>,
349    pub activity_type: Option<String>,
350    pub activity_class: Option<String>,
351    pub attempt_number: Option<u64>,
352    pub failure_id: Option<String>,
353    pub failure_category: Option<String>,
354    pub timeout_kind: Option<String>,
355    pub non_retryable: bool,
356    pub exception_type: Option<String>,
357    pub exception_class: Option<String>,
358    pub code: Option<Value>,
359    pub exception: Option<Value>,
360}
361
362/// Stable terminal categories returned when an awaited child does not succeed.
363#[derive(Clone, Copy, Debug, PartialEq, Eq)]
364pub enum ChildWorkflowFailureKind {
365    Failed,
366    Cancelled,
367    Terminated,
368}
369
370/// A stable, machine-readable child workflow failure delivered to its parent.
371///
372/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
373/// of parsing the display message. Child and parent identifiers retain the
374/// relationship recorded in durable history across worker restarts.
375#[derive(Clone, Debug, Error)]
376#[error("child workflow failed ({reason}): {message}")]
377pub struct ChildWorkflowFailure {
378    pub kind: ChildWorkflowFailureKind,
379    pub reason: String,
380    pub message: String,
381    pub parent_workflow_id: Option<String>,
382    pub parent_workflow_run_id: Option<String>,
383    pub child_workflow_id: Option<String>,
384    pub child_workflow_run_id: Option<String>,
385    pub child_workflow_type: Option<String>,
386    pub failure_id: Option<String>,
387    pub failure_category: Option<String>,
388    pub exception_type: Option<String>,
389    pub exception_class: Option<String>,
390    pub non_retryable: bool,
391    pub code: Option<Value>,
392    pub exception: Option<Value>,
393}
394
395/// The identity of one durable workflow execution.
396#[derive(Clone, Debug, PartialEq, Eq)]
397pub struct WorkflowIdentity {
398    pub workflow_id: Option<String>,
399    pub run_id: Option<String>,
400}
401
402/// A successful child result together with its durable parent-child identity.
403#[derive(Clone, Debug, PartialEq)]
404pub struct ChildWorkflowResult {
405    pub parent: WorkflowIdentity,
406    pub child: WorkflowIdentity,
407    pub child_workflow_type: Option<String>,
408    pub result: Value,
409}
410
411/// Server behavior when a parent closes while its child is still open.
412#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
413pub enum ParentClosePolicy {
414    #[default]
415    Abandon,
416    RequestCancel,
417    Terminate,
418}
419
420impl ParentClosePolicy {
421    fn as_str(self) -> &'static str {
422        match self {
423            Self::Abandon => "abandon",
424            Self::RequestCancel => "request_cancel",
425            Self::Terminate => "terminate",
426        }
427    }
428}
429
430/// Durable retry policy for one child workflow invocation.
431#[derive(Clone, Debug, Default, PartialEq, Eq)]
432pub struct ChildWorkflowRetryPolicy {
433    pub max_attempts: Option<u32>,
434    pub backoff_seconds: Vec<u64>,
435    pub non_retryable_error_types: Vec<String>,
436}
437
438/// Options recorded with a child-workflow command.
439///
440/// The task queue is mandatory so routing is explicit and replay-stable.
441#[derive(Clone, Debug, PartialEq, Eq)]
442pub struct ChildWorkflowOptions {
443    pub task_queue: String,
444    pub parent_close_policy: ParentClosePolicy,
445    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
446    pub execution_timeout_seconds: Option<u64>,
447    pub run_timeout_seconds: Option<u64>,
448}
449
450impl ChildWorkflowOptions {
451    pub fn new(task_queue: impl Into<String>) -> Self {
452        Self {
453            task_queue: task_queue.into(),
454            parent_close_policy: ParentClosePolicy::Abandon,
455            retry_policy: None,
456            execution_timeout_seconds: None,
457            run_timeout_seconds: None,
458        }
459    }
460
461    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
462        self.parent_close_policy = policy;
463        self
464    }
465
466    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
467        self.retry_policy = Some(policy);
468        self
469    }
470
471    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
472        self.execution_timeout_seconds = Some(seconds);
473        self
474    }
475
476    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
477        self.run_timeout_seconds = Some(seconds);
478        self
479    }
480}
481
482/// Backoff intervals for one durable activity retry policy.
483#[derive(Clone, Debug, PartialEq, Eq)]
484pub enum ActivityBackoff {
485    /// Use these intervals between attempts. The server repeats the final
486    /// interval if the retry budget contains more attempts than entries.
487    Explicit(Vec<Duration>),
488    /// Generate one interval for every retry using integer exponential growth.
489    Exponential {
490        initial_interval: Duration,
491        coefficient: u32,
492        maximum_interval: Option<Duration>,
493    },
494}
495
496/// Durable server-side retry policy for one activity execution.
497#[derive(Clone, Debug, Default, PartialEq, Eq)]
498pub struct ActivityRetryPolicy {
499    pub max_attempts: Option<u32>,
500    pub backoff: Option<ActivityBackoff>,
501    pub non_retryable_error_types: Vec<String>,
502}
503
504impl ActivityRetryPolicy {
505    /// Start a policy with a finite attempt budget, including the first attempt.
506    pub fn new(max_attempts: u32) -> Self {
507        Self {
508            max_attempts: Some(max_attempts),
509            ..Self::default()
510        }
511    }
512
513    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
514        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
515        self
516    }
517
518    pub fn exponential_backoff(
519        mut self,
520        initial_interval: Duration,
521        coefficient: u32,
522        maximum_interval: Option<Duration>,
523    ) -> Self {
524        self.backoff = Some(ActivityBackoff::Exponential {
525            initial_interval,
526            coefficient,
527            maximum_interval,
528        });
529        self
530    }
531
532    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
533        self.non_retryable_error_types.push(error_type.into());
534        self
535    }
536
537    pub fn non_retryable_error_types(
538        mut self,
539        error_types: impl IntoIterator<Item = impl Into<String>>,
540    ) -> Self {
541        self.non_retryable_error_types
542            .extend(error_types.into_iter().map(Into::into));
543        self
544    }
545}
546
547/// Options recorded atomically on one deterministic `schedule_activity` command.
548///
549/// Durations are rounded up to whole seconds when encoded, so the server never
550/// receives a shorter timeout or backoff than the caller requested.
551#[derive(Clone, Debug, Default, PartialEq, Eq)]
552pub struct ActivityOptions {
553    pub task_queue: Option<String>,
554    pub retry_policy: Option<ActivityRetryPolicy>,
555    pub start_to_close_timeout: Option<Duration>,
556    pub schedule_to_start_timeout: Option<Duration>,
557    pub schedule_to_close_timeout: Option<Duration>,
558    pub heartbeat_timeout: Option<Duration>,
559}
560
561impl ActivityOptions {
562    pub fn new() -> Self {
563        Self::default()
564    }
565
566    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
567        self.task_queue = Some(task_queue.into());
568        self
569    }
570
571    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
572        self.retry_policy = Some(policy);
573        self
574    }
575
576    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
577        self.start_to_close_timeout = Some(timeout);
578        self
579    }
580
581    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
582        self.schedule_to_start_timeout = Some(timeout);
583        self
584    }
585
586    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
587        self.schedule_to_close_timeout = Some(timeout);
588        self
589    }
590
591    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
592        self.heartbeat_timeout = Some(timeout);
593        self
594    }
595
596    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
597        if self
598            .task_queue
599            .as_deref()
600            .is_some_and(|queue| queue.trim().is_empty())
601        {
602            return Err(ActivityOptionsError::new(
603                ActivityOptionsErrorKind::EmptyTaskQueue,
604                Some("task_queue"),
605                "task_queue must not be empty",
606            ));
607        }
608
609        for (field, value) in [
610            ("start_to_close_timeout", self.start_to_close_timeout),
611            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
612            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
613            ("heartbeat_timeout", self.heartbeat_timeout),
614        ] {
615            if value.is_some_and(|value| value.is_zero()) {
616                return Err(ActivityOptionsError::new(
617                    ActivityOptionsErrorKind::TimeoutNotPositive,
618                    Some(field),
619                    format!("{field} must be positive"),
620                ));
621            }
622        }
623
624        validate_timeout_order(
625            "heartbeat_timeout",
626            self.heartbeat_timeout,
627            "start_to_close_timeout",
628            self.start_to_close_timeout,
629        )?;
630        validate_timeout_order(
631            "start_to_close_timeout",
632            self.start_to_close_timeout,
633            "schedule_to_close_timeout",
634            self.schedule_to_close_timeout,
635        )?;
636        validate_timeout_order(
637            "schedule_to_start_timeout",
638            self.schedule_to_start_timeout,
639            "schedule_to_close_timeout",
640            self.schedule_to_close_timeout,
641        )?;
642
643        Ok(ValidatedActivityOptions {
644            task_queue: self.task_queue.clone(),
645            retry_policy: self
646                .retry_policy
647                .as_ref()
648                .map(validate_activity_retry_policy)
649                .transpose()?,
650            start_to_close_timeout: timeout_seconds(
651                "start_to_close_timeout",
652                self.start_to_close_timeout,
653            )?,
654            schedule_to_start_timeout: timeout_seconds(
655                "schedule_to_start_timeout",
656                self.schedule_to_start_timeout,
657            )?,
658            schedule_to_close_timeout: timeout_seconds(
659                "schedule_to_close_timeout",
660                self.schedule_to_close_timeout,
661            )?,
662            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
663        })
664    }
665}
666
667#[derive(Clone, Debug)]
668struct ValidatedActivityOptions {
669    task_queue: Option<String>,
670    retry_policy: Option<Value>,
671    start_to_close_timeout: Option<u64>,
672    schedule_to_start_timeout: Option<u64>,
673    schedule_to_close_timeout: Option<u64>,
674    heartbeat_timeout: Option<u64>,
675}
676
677fn validate_timeout_order(
678    smaller_name: &'static str,
679    smaller: Option<Duration>,
680    larger_name: &'static str,
681    larger: Option<Duration>,
682) -> std::result::Result<(), ActivityOptionsError> {
683    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
684        return Err(ActivityOptionsError::new(
685            ActivityOptionsErrorKind::TimeoutOrder,
686            Some(smaller_name),
687            format!("{smaller_name} must be <= {larger_name}"),
688        ));
689    }
690    Ok(())
691}
692
693fn timeout_seconds(
694    field: &'static str,
695    value: Option<Duration>,
696) -> std::result::Result<Option<u64>, ActivityOptionsError> {
697    value
698        .map(|value| {
699            activity_protocol_seconds(value).ok_or_else(|| {
700                ActivityOptionsError::new(
701                    ActivityOptionsErrorKind::TimeoutOverflow,
702                    Some(field),
703                    format!("{field} is too large for the worker protocol"),
704                )
705            })
706        })
707        .transpose()
708}
709
710fn duration_seconds_ceil(value: Duration) -> Option<u64> {
711    value
712        .as_secs()
713        .checked_add(u64::from(value.subsec_nanos() > 0))
714}
715
716fn activity_protocol_seconds(value: Duration) -> Option<u64> {
717    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
718}
719
720fn validate_activity_retry_policy(
721    policy: &ActivityRetryPolicy,
722) -> std::result::Result<Value, ActivityOptionsError> {
723    if policy.max_attempts.is_none()
724        && policy.backoff.is_none()
725        && policy.non_retryable_error_types.is_empty()
726    {
727        return Err(ActivityOptionsError::new(
728            ActivityOptionsErrorKind::EmptyRetryPolicy,
729            Some("retry_policy"),
730            "retry_policy must configure at least one field",
731        ));
732    }
733    if policy.max_attempts == Some(0) {
734        return Err(ActivityOptionsError::new(
735            ActivityOptionsErrorKind::InvalidMaxAttempts,
736            Some("retry_policy.max_attempts"),
737            "max_attempts must be >= 1",
738        ));
739    }
740    if policy
741        .non_retryable_error_types
742        .iter()
743        .any(|error_type| error_type.trim().is_empty())
744    {
745        return Err(ActivityOptionsError::new(
746            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
747            Some("retry_policy.non_retryable_error_types"),
748            "non_retryable_error_types must not contain empty values",
749        ));
750    }
751
752    let backoff_seconds = match &policy.backoff {
753        None => None,
754        Some(backoff) => {
755            let max_attempts = policy.max_attempts.ok_or_else(|| {
756                ActivityOptionsError::new(
757                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
758                    Some("retry_policy.backoff"),
759                    "backoff requires max_attempts",
760                )
761            })?;
762            let retry_count = max_attempts.saturating_sub(1) as usize;
763            let intervals = match backoff {
764                ActivityBackoff::Explicit(intervals) => {
765                    if intervals.len() > retry_count {
766                        return Err(ActivityOptionsError::new(
767                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
768                            Some("retry_policy.backoff"),
769                            "backoff interval count must not exceed max_attempts - 1",
770                        ));
771                    }
772                    intervals.clone()
773                }
774                ActivityBackoff::Exponential {
775                    initial_interval,
776                    coefficient,
777                    maximum_interval,
778                } => {
779                    if *coefficient < 1 {
780                        return Err(ActivityOptionsError::new(
781                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
782                            Some("retry_policy.backoff.coefficient"),
783                            "backoff coefficient must be >= 1",
784                        ));
785                    }
786                    if retry_count > 10_000 {
787                        return Err(ActivityOptionsError::new(
788                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
789                            Some("retry_policy.max_attempts"),
790                            "generated backoff supports at most 10000 retry intervals",
791                        ));
792                    }
793                    let mut current = *initial_interval;
794                    let mut intervals = Vec::with_capacity(retry_count);
795                    for _ in 0..retry_count {
796                        let interval = maximum_interval
797                            .map(|maximum| current.min(maximum))
798                            .unwrap_or(current);
799                        intervals.push(interval);
800                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
801                            break;
802                        }
803                        current = current.checked_mul(*coefficient).ok_or_else(|| {
804                            ActivityOptionsError::new(
805                                ActivityOptionsErrorKind::BackoffOverflow,
806                                Some("retry_policy.backoff"),
807                                "generated backoff interval overflowed",
808                            )
809                        })?;
810                    }
811                    intervals
812                }
813            };
814            Some(
815                intervals
816                    .into_iter()
817                    .map(|interval| {
818                        activity_protocol_seconds(interval).ok_or_else(|| {
819                            ActivityOptionsError::new(
820                                ActivityOptionsErrorKind::BackoffOverflow,
821                                Some("retry_policy.backoff"),
822                                "backoff interval is too large for the worker protocol",
823                            )
824                        })
825                    })
826                    .collect::<std::result::Result<Vec<_>, _>>()?,
827            )
828        }
829    };
830
831    let mut encoded = serde_json::Map::new();
832    if let Some(max_attempts) = policy.max_attempts {
833        encoded.insert("max_attempts".to_string(), json!(max_attempts));
834    }
835    if let Some(backoff_seconds) = backoff_seconds {
836        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
837    }
838    if !policy.non_retryable_error_types.is_empty() {
839        let mut canonical_error_types = Vec::new();
840        for error_type in policy
841            .non_retryable_error_types
842            .iter()
843            .map(|error_type| error_type.trim())
844        {
845            if !canonical_error_types.contains(&error_type) {
846                canonical_error_types.push(error_type);
847            }
848        }
849        encoded.insert(
850            "non_retryable_error_types".to_string(),
851            json!(canonical_error_types),
852        );
853    }
854    Ok(Value::Object(encoded))
855}
856
857/// A stable, machine-readable failure raised when workflow code no longer
858/// reconstructs the durable command stream recorded in history.
859#[derive(Clone, Debug, Error)]
860#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
861pub struct ReplayFailure {
862    pub reason: String,
863    pub sequence: Option<u64>,
864    pub expected: Option<String>,
865    pub actual: Option<String>,
866    pub message: String,
867}
868
869impl ReplayFailure {
870    fn new(
871        reason: impl Into<String>,
872        sequence: Option<u64>,
873        expected: Option<String>,
874        actual: Option<String>,
875        message: impl Into<String>,
876    ) -> Self {
877        Self {
878            reason: reason.into(),
879            sequence,
880            expected,
881            actual,
882            message: message.into(),
883        }
884    }
885}
886
887/// A stable, machine-readable workflow query or query-task settlement failure.
888#[derive(Clone, Debug, Error)]
889#[error("query failed ({reason}, HTTP {status}): {message}")]
890pub struct QueryFailure {
891    pub status: u16,
892    pub reason: String,
893    pub message: String,
894    pub body: Value,
895}
896
897/// A stable failure returned when a server rejects an SDK protocol version.
898#[derive(Clone, Debug, Error)]
899#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
900pub struct ProtocolFailure {
901    pub status: u16,
902    pub reason: String,
903    pub message: String,
904    pub supported_version: Option<String>,
905    pub requested_version: Option<String>,
906    pub body: Value,
907}
908
909#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
910pub struct PayloadEnvelope {
911    pub codec: String,
912    pub blob: String,
913}
914
915impl PayloadEnvelope {
916    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
917        encode_payload(value, DEFAULT_CODEC)
918    }
919
920    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
921        encode_payload(value, JSON_CODEC)
922    }
923}
924
925pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
926    let value = serde_json::to_value(value)?;
927    let blob = encode_value_blob(&value, codec)?;
928
929    Ok(PayloadEnvelope {
930        codec: codec.to_string(),
931        blob,
932    })
933}
934
935pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
936    let value = decode_blob(&envelope.blob, &envelope.codec)?;
937    Ok(serde_json::from_value(value)?)
938}
939
940fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
941    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
942}
943
944fn encode_value_blob(value: &Value, codec: &str) -> Result<String> {
945    match codec {
946        JSON_CODEC => Ok(serde_json::to_string(value)?),
947        DEFAULT_CODEC => encode_avro_generic(value),
948        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
949    }
950}
951
952fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
953    if value.is_null() {
954        return Ok(Value::Null);
955    }
956
957    if let Some(object) = value.as_object() {
958        if let (Some(codec), Some(blob)) = (
959            object.get("codec").and_then(Value::as_str),
960            object.get("blob").and_then(Value::as_str),
961        ) {
962            return decode_blob(blob, codec);
963        }
964    }
965
966    if let Some(blob) = value.as_str() {
967        return decode_blob(blob, fallback_codec);
968    }
969
970    Ok(value.clone())
971}
972
973fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
974    match codec {
975        JSON_CODEC => Ok(serde_json::from_str(blob)?),
976        DEFAULT_CODEC => decode_avro_generic(blob),
977        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
978    }
979}
980
981fn encode_avro_generic(value: &Value) -> Result<String> {
982    let json = serde_json::to_string(value)?;
983    let datum = to_value(AvroPayload {
984        json,
985        version: AVRO_PAYLOAD_VERSION,
986    })
987    .map_err(|err| Error::Codec(format!("could not convert avro generic wrapper: {err}")))?;
988    let datum = to_avro_datum(avro_payload_schema()?, datum)
989        .map_err(|err| Error::Codec(format!("could not encode avro generic wrapper: {err}")))?;
990
991    let mut bytes = Vec::with_capacity(datum.len() + 1);
992    bytes.push(0x00);
993    bytes.extend_from_slice(&datum);
994    Ok(BASE64.encode(bytes))
995}
996
997fn decode_avro_generic(blob: &str) -> Result<Value> {
998    let bytes = BASE64
999        .decode(blob)
1000        .map_err(|err| Error::Codec(format!("invalid avro base64 payload: {err}")))?;
1001
1002    if bytes.is_empty() {
1003        return Err(Error::Codec("avro payload is empty".to_string()));
1004    }
1005
1006    match bytes[0] {
1007        0x00 => {}
1008        0x01 => {
1009            return Err(Error::Codec(
1010                "typed avro payloads require a schema context; v1 supports the generic wrapper"
1011                    .to_string(),
1012            ));
1013        }
1014        other => {
1015            return Err(Error::Codec(format!(
1016                "unknown avro payload prefix 0x{other:02x}"
1017            )));
1018        }
1019    }
1020
1021    let mut datum = &bytes[1..];
1022    let datum = from_avro_datum(avro_payload_schema()?, &mut datum, None)
1023        .map_err(|err| Error::Codec(format!("could not decode avro generic wrapper: {err}")))?;
1024    let payload: AvroPayload = from_value(&datum)
1025        .map_err(|err| Error::Codec(format!("invalid avro generic wrapper record: {err}")))?;
1026
1027    if payload.version != AVRO_PAYLOAD_VERSION {
1028        return Err(Error::Codec(format!(
1029            "unsupported avro generic wrapper version {}",
1030            payload.version
1031        )));
1032    }
1033
1034    Ok(serde_json::from_str(&payload.json)?)
1035}
1036
1037#[derive(Debug, Serialize, Deserialize)]
1038struct AvroPayload {
1039    json: String,
1040    version: i32,
1041}
1042
1043fn avro_payload_schema() -> Result<&'static Schema> {
1044    match AVRO_PAYLOAD_SCHEMA.get_or_init(|| {
1045        Schema::parse_str(AVRO_PAYLOAD_SCHEMA_JSON)
1046            .map_err(|err| format!("could not parse avro payload schema: {err}"))
1047    }) {
1048        Ok(schema) => Ok(schema),
1049        Err(message) => Err(Error::Codec(message.clone())),
1050    }
1051}
1052
1053#[derive(Clone, Debug)]
1054pub struct Client {
1055    http: reqwest::Client,
1056    base_url: String,
1057    token: Option<String>,
1058    control_token: Option<String>,
1059    worker_token: Option<String>,
1060    namespace: String,
1061}
1062
1063impl Client {
1064    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1065        Self::builder(base_url).build()
1066    }
1067
1068    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1069        ClientBuilder {
1070            base_url: base_url.into(),
1071            token: None,
1072            control_token: None,
1073            worker_token: None,
1074            namespace: "default".to_string(),
1075            timeout: Duration::from_secs(60),
1076        }
1077    }
1078
1079    pub async fn health(&self) -> Result<Value> {
1080        self.request_json(
1081            reqwest::Method::GET,
1082            "/health",
1083            RequestProtocol::ControlPlane,
1084            Option::<&Value>::None,
1085        )
1086        .await
1087    }
1088
1089    pub async fn cluster_info(&self) -> Result<Value> {
1090        self.request_json(
1091            reqwest::Method::GET,
1092            "/cluster/info",
1093            RequestProtocol::ControlPlane,
1094            Option::<&Value>::None,
1095        )
1096        .await
1097    }
1098
1099    pub async fn start_workflow<T: Serialize>(
1100        &self,
1101        workflow_type: &str,
1102        task_queue: &str,
1103        workflow_id: &str,
1104        input: T,
1105    ) -> Result<WorkflowHandle> {
1106        self.start_workflow_with_options(
1107            workflow_type,
1108            task_queue,
1109            workflow_id,
1110            WorkflowStartOptions::default(),
1111            input,
1112        )
1113        .await
1114    }
1115
1116    /// Start a workflow with explicit server-enforced execution and run
1117    /// deadlines.
1118    pub async fn start_workflow_with_options<T: Serialize>(
1119        &self,
1120        workflow_type: &str,
1121        task_queue: &str,
1122        workflow_id: &str,
1123        options: WorkflowStartOptions,
1124        input: T,
1125    ) -> Result<WorkflowHandle> {
1126        options.validate()?;
1127        let input = serde_json::to_value(input)?;
1128        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1129        let body = json!({
1130            "workflow_id": workflow_id,
1131            "workflow_type": workflow_type,
1132            "task_queue": task_queue,
1133            "input": input_envelope,
1134            "execution_timeout_seconds": options.execution_timeout_seconds,
1135            "run_timeout_seconds": options.run_timeout_seconds
1136        });
1137
1138        let data: Value = self
1139            .request_json(
1140                reqwest::Method::POST,
1141                "/workflows",
1142                RequestProtocol::ControlPlane,
1143                Some(&body),
1144            )
1145            .await?;
1146
1147        Ok(WorkflowHandle {
1148            client: self.clone(),
1149            workflow_id: data
1150                .get("workflow_id")
1151                .and_then(Value::as_str)
1152                .unwrap_or(workflow_id)
1153                .to_string(),
1154            run_id: data
1155                .get("run_id")
1156                .and_then(Value::as_str)
1157                .map(str::to_string),
1158            workflow_type: data
1159                .get("workflow_type")
1160                .and_then(Value::as_str)
1161                .unwrap_or(workflow_type)
1162                .to_string(),
1163        })
1164    }
1165
1166    pub async fn signal_workflow<T: Serialize>(
1167        &self,
1168        workflow_id: &str,
1169        signal_name: &str,
1170        input: T,
1171    ) -> Result<Value> {
1172        let input = serde_json::to_value(input)?;
1173        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1174        let body = json!({
1175            "input": input_envelope
1176        });
1177        let path = format!("/workflows/{workflow_id}/signal/{signal_name}");
1178        self.request_json(
1179            reqwest::Method::POST,
1180            &path,
1181            RequestProtocol::ControlPlane,
1182            Some(&body),
1183        )
1184        .await
1185    }
1186
1187    /// Request cooperative cancellation of the current run for an instance.
1188    pub async fn cancel_workflow(
1189        &self,
1190        workflow_id: &str,
1191        options: WorkflowCommandOptions,
1192    ) -> Result<WorkflowCommandResult> {
1193        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1194            .await
1195    }
1196
1197    /// Request cooperative cancellation only if `run_id` is still current.
1198    pub async fn cancel_workflow_run(
1199        &self,
1200        workflow_id: &str,
1201        run_id: &str,
1202        options: WorkflowCommandOptions,
1203    ) -> Result<WorkflowCommandResult> {
1204        self.workflow_command(
1205            workflow_id,
1206            Some(run_id),
1207            WorkflowCommandKind::Cancel,
1208            options,
1209        )
1210        .await
1211    }
1212
1213    /// Forcefully terminate the current run for an instance.
1214    pub async fn terminate_workflow(
1215        &self,
1216        workflow_id: &str,
1217        options: WorkflowCommandOptions,
1218    ) -> Result<WorkflowCommandResult> {
1219        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1220            .await
1221    }
1222
1223    /// Forcefully terminate only if `run_id` is still current.
1224    pub async fn terminate_workflow_run(
1225        &self,
1226        workflow_id: &str,
1227        run_id: &str,
1228        options: WorkflowCommandOptions,
1229    ) -> Result<WorkflowCommandResult> {
1230        self.workflow_command(
1231            workflow_id,
1232            Some(run_id),
1233            WorkflowCommandKind::Terminate,
1234            options,
1235        )
1236        .await
1237    }
1238
1239    async fn workflow_command(
1240        &self,
1241        workflow_id: &str,
1242        run_id: Option<&str>,
1243        command: WorkflowCommandKind,
1244        options: WorkflowCommandOptions,
1245    ) -> Result<WorkflowCommandResult> {
1246        let path = match run_id {
1247            Some(run_id) => format!(
1248                "/workflows/{workflow_id}/runs/{run_id}/{}",
1249                command.as_str()
1250            ),
1251            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1252        };
1253        let data = match self
1254            .request_json(
1255                reqwest::Method::POST,
1256                &path,
1257                RequestProtocol::ControlPlane,
1258                Some(&options),
1259            )
1260            .await
1261        {
1262            Ok(data) => data,
1263            Err(Error::Http { status, body }) => {
1264                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1265                    command,
1266                    status,
1267                    body,
1268                    workflow_id,
1269                    run_id,
1270                )));
1271            }
1272            Err(error) => return Err(error),
1273        };
1274
1275        Ok(workflow_command_result(command, data, workflow_id, run_id))
1276    }
1277
1278    /// Execute a named, read-only query against a running or completed workflow.
1279    ///
1280    /// Arguments and results use the platform payload envelope. Server and
1281    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1282    /// reason, HTTP status, and original response body.
1283    pub async fn query_workflow<T: Serialize>(
1284        &self,
1285        workflow_id: &str,
1286        query_name: &str,
1287        input: T,
1288    ) -> Result<Value> {
1289        let input = serde_json::to_value(input)?;
1290        let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1291        let body = json!({
1292            "input": input_envelope
1293        });
1294        let path = format!("/workflows/{workflow_id}/query/{query_name}");
1295        let response: Value = match self
1296            .request_json(
1297                reqwest::Method::POST,
1298                &path,
1299                RequestProtocol::ControlPlane,
1300                Some(&body),
1301            )
1302            .await
1303        {
1304            Ok(response) => response,
1305            Err(Error::Http { status, body }) => {
1306                return Err(Error::QueryFailed(query_failure(status, body)));
1307            }
1308            Err(error) => return Err(error),
1309        };
1310
1311        if let Some(envelope) = response
1312            .get("result_envelope")
1313            .filter(|envelope| !envelope.is_null())
1314        {
1315            return decode_wire_value(envelope, DEFAULT_CODEC);
1316        }
1317
1318        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1319    }
1320
1321    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
1322        let path = format!("/workflows/{workflow_id}");
1323        let mut data: WorkflowDescription = self
1324            .request_json(
1325                reqwest::Method::GET,
1326                &path,
1327                RequestProtocol::ControlPlane,
1328                Option::<&Value>::None,
1329            )
1330            .await?;
1331        data.decode_payloads()?;
1332        Ok(data)
1333    }
1334
1335    /// Describe one selected run, including historical terminal runs.
1336    pub async fn describe_workflow_run(
1337        &self,
1338        workflow_id: &str,
1339        run_id: &str,
1340    ) -> Result<WorkflowDescription> {
1341        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
1342        let mut data: WorkflowDescription = self
1343            .request_json(
1344                reqwest::Method::GET,
1345                &path,
1346                RequestProtocol::ControlPlane,
1347                Option::<&Value>::None,
1348            )
1349            .await?;
1350        data.decode_payloads()?;
1351        Ok(data)
1352    }
1353
1354    pub async fn register_worker(
1355        &self,
1356        worker_id: &str,
1357        task_queue: &str,
1358        supported_workflow_types: Vec<String>,
1359        supported_activity_types: Vec<String>,
1360        max_concurrent_workflow_tasks: usize,
1361        max_concurrent_activity_tasks: usize,
1362    ) -> Result<RegisterWorkerResponse> {
1363        self.register_worker_with_capabilities(
1364            worker_id,
1365            task_queue,
1366            supported_workflow_types,
1367            supported_activity_types,
1368            max_concurrent_workflow_tasks,
1369            max_concurrent_activity_tasks,
1370            Vec::new(),
1371        )
1372        .await
1373    }
1374
1375    /// Register a worker and explicitly advertise additive worker capabilities.
1376    pub async fn register_worker_with_capabilities(
1377        &self,
1378        worker_id: &str,
1379        task_queue: &str,
1380        supported_workflow_types: Vec<String>,
1381        supported_activity_types: Vec<String>,
1382        max_concurrent_workflow_tasks: usize,
1383        max_concurrent_activity_tasks: usize,
1384        capabilities: Vec<String>,
1385    ) -> Result<RegisterWorkerResponse> {
1386        let body = json!({
1387            "worker_id": worker_id,
1388            "task_queue": task_queue,
1389            "runtime": "rust",
1390            "sdk_version": SDK_VERSION,
1391            "supported_workflow_types": supported_workflow_types,
1392            "supported_activity_types": supported_activity_types,
1393            "capabilities": capabilities,
1394            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
1395            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
1396        });
1397
1398        self.request_json(
1399            reqwest::Method::POST,
1400            "/worker/register",
1401            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1402            Some(&body),
1403        )
1404        .await
1405    }
1406
1407    /// Long-poll for an ephemeral, read-only workflow query task.
1408    pub async fn poll_query_task(
1409        &self,
1410        worker_id: &str,
1411        task_queue: &str,
1412        timeout: Duration,
1413    ) -> Result<Option<QueryTask>> {
1414        Ok(self
1415            .poll_query_task_response(worker_id, task_queue, timeout)
1416            .await?
1417            .task)
1418    }
1419
1420    /// Poll a query task while preserving server stop and drain metadata.
1421    pub async fn poll_query_task_response(
1422        &self,
1423        worker_id: &str,
1424        task_queue: &str,
1425        timeout: Duration,
1426    ) -> Result<PollQueryTaskResponse> {
1427        let timeout_seconds = long_poll_timeout_seconds(timeout);
1428        let body = json!({
1429            "worker_id": worker_id,
1430            "task_queue": task_queue,
1431            "poll_request_id": unique_request_id("rust-query-poll"),
1432            "timeout_seconds": timeout_seconds,
1433        });
1434        worker_poll_response(
1435            self.request_json_with_timeout(
1436                reqwest::Method::POST,
1437                "/worker/query-tasks/poll",
1438                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1439                Some(&body),
1440                timeout + Duration::from_secs(5),
1441            )
1442            .await,
1443        )
1444    }
1445
1446    /// Complete a query task without appending workflow history.
1447    pub async fn complete_query_task(
1448        &self,
1449        query_task_id: &str,
1450        lease_owner: &str,
1451        query_task_attempt: u64,
1452        result: Value,
1453        codec: &str,
1454    ) -> Result<Value> {
1455        let result_envelope = encode_value_envelope(&result, codec)?;
1456        self.complete_query_task_with_envelope(
1457            query_task_id,
1458            lease_owner,
1459            query_task_attempt,
1460            result,
1461            result_envelope,
1462        )
1463        .await
1464    }
1465
1466    async fn complete_query_task_with_envelope(
1467        &self,
1468        query_task_id: &str,
1469        lease_owner: &str,
1470        query_task_attempt: u64,
1471        result: Value,
1472        result_envelope: Value,
1473    ) -> Result<Value> {
1474        let body = json!({
1475            "lease_owner": lease_owner,
1476            "query_task_attempt": query_task_attempt,
1477            "result": result,
1478            "result_envelope": result_envelope,
1479        });
1480        let path = format!("/worker/query-tasks/{query_task_id}/complete");
1481        let response = self
1482            .request_json(
1483                reqwest::Method::POST,
1484                &path,
1485                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1486                Some(&body),
1487            )
1488            .await;
1489        query_task_response(response)
1490    }
1491
1492    /// Report a stable machine-readable query-task failure.
1493    pub async fn fail_query_task(
1494        &self,
1495        query_task_id: &str,
1496        lease_owner: &str,
1497        query_task_attempt: u64,
1498        message: impl Into<String>,
1499        reason: impl Into<String>,
1500        failure_type: impl Into<String>,
1501    ) -> Result<Value> {
1502        let body = json!({
1503            "lease_owner": lease_owner,
1504            "query_task_attempt": query_task_attempt,
1505            "failure": {
1506                "message": message.into(),
1507                "reason": reason.into(),
1508                "type": failure_type.into(),
1509            }
1510        });
1511        let path = format!("/worker/query-tasks/{query_task_id}/fail");
1512        let response = self
1513            .request_json(
1514                reqwest::Method::POST,
1515                &path,
1516                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1517                Some(&body),
1518            )
1519            .await;
1520        query_task_response(response)
1521    }
1522
1523    pub async fn heartbeat_worker(
1524        &self,
1525        worker_id: &str,
1526        workflow_available: usize,
1527        activity_available: usize,
1528    ) -> Result<Value> {
1529        let body = json!({
1530            "worker_id": worker_id,
1531            "task_slots": {
1532                "workflow_available": workflow_available,
1533                "activity_available": activity_available
1534            },
1535            "process_metrics": {
1536                "process_id": std::process::id(),
1537                "process_uptime_seconds": 0
1538            }
1539        });
1540
1541        self.request_json(
1542            reqwest::Method::POST,
1543            "/worker/heartbeat",
1544            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1545            Some(&body),
1546        )
1547        .await
1548    }
1549
1550    pub async fn poll_workflow_task(
1551        &self,
1552        worker_id: &str,
1553        task_queue: &str,
1554        timeout: Duration,
1555    ) -> Result<Option<WorkflowTask>> {
1556        Ok(self
1557            .poll_workflow_task_response(worker_id, task_queue, timeout)
1558            .await?
1559            .task)
1560    }
1561
1562    pub async fn poll_workflow_task_response(
1563        &self,
1564        worker_id: &str,
1565        task_queue: &str,
1566        timeout: Duration,
1567    ) -> Result<PollWorkflowTaskResponse> {
1568        let body = json!({
1569            "worker_id": worker_id,
1570            "task_queue": task_queue,
1571            "timeout_seconds": long_poll_timeout_seconds(timeout),
1572        });
1573        let mut data: PollWorkflowTaskResponse = worker_poll_response(
1574            self.request_json_with_timeout(
1575                reqwest::Method::POST,
1576                "/worker/workflow-tasks/poll",
1577                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1578                Some(&body),
1579                timeout + Duration::from_secs(5),
1580            )
1581            .await,
1582        )?;
1583
1584        if let Some(task) = data.task.as_mut() {
1585            self.fetch_remaining_workflow_history(worker_id, task)
1586                .await?;
1587        }
1588
1589        Ok(data)
1590    }
1591
1592    async fn fetch_remaining_workflow_history(
1593        &self,
1594        worker_id: &str,
1595        task: &mut WorkflowTask,
1596    ) -> Result<()> {
1597        let mut next_token = task.next_history_page_token.clone();
1598
1599        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
1600            let lease_owner = task
1601                .lease_owner
1602                .clone()
1603                .unwrap_or_else(|| worker_id.to_string());
1604            let page = self
1605                .workflow_task_history_page(
1606                    &task.task_id,
1607                    &lease_owner,
1608                    task.workflow_task_attempt,
1609                    &token,
1610                )
1611                .await?;
1612
1613            task.append_history_page(page);
1614
1615            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
1616                return Err(Error::Codec(
1617                    "workflow history pagination returned the same page token".to_string(),
1618                ));
1619            }
1620
1621            next_token = task.next_history_page_token.clone();
1622        }
1623
1624        Ok(())
1625    }
1626
1627    async fn workflow_task_history_page(
1628        &self,
1629        task_id: &str,
1630        lease_owner: &str,
1631        workflow_task_attempt: u64,
1632        next_history_page_token: &str,
1633    ) -> Result<WorkflowTaskHistoryPage> {
1634        let body = json!({
1635            "lease_owner": lease_owner,
1636            "workflow_task_attempt": workflow_task_attempt,
1637            "next_history_page_token": next_history_page_token
1638        });
1639        let path = format!("/worker/workflow-tasks/{task_id}/history");
1640
1641        self.request_json(
1642            reqwest::Method::POST,
1643            &path,
1644            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1645            Some(&body),
1646        )
1647        .await
1648    }
1649
1650    pub async fn complete_workflow_task(
1651        &self,
1652        task_id: &str,
1653        lease_owner: &str,
1654        workflow_task_attempt: u64,
1655        commands: Vec<Value>,
1656    ) -> Result<Value> {
1657        let body = json!({
1658            "lease_owner": lease_owner,
1659            "workflow_task_attempt": workflow_task_attempt,
1660            "commands": commands
1661        });
1662        let path = format!("/worker/workflow-tasks/{task_id}/complete");
1663        self.request_json(
1664            reqwest::Method::POST,
1665            &path,
1666            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1667            Some(&body),
1668        )
1669        .await
1670    }
1671
1672    pub async fn fail_workflow_task(
1673        &self,
1674        task_id: &str,
1675        lease_owner: &str,
1676        workflow_task_attempt: u64,
1677        message: impl Into<String>,
1678    ) -> Result<Value> {
1679        self.fail_workflow_task_with_type(
1680            task_id,
1681            lease_owner,
1682            workflow_task_attempt,
1683            message,
1684            "RustWorkflowTaskFailure",
1685        )
1686        .await
1687    }
1688
1689    async fn fail_workflow_task_with_type(
1690        &self,
1691        task_id: &str,
1692        lease_owner: &str,
1693        workflow_task_attempt: u64,
1694        message: impl Into<String>,
1695        failure_type: &str,
1696    ) -> Result<Value> {
1697        let body = json!({
1698            "lease_owner": lease_owner,
1699            "workflow_task_attempt": workflow_task_attempt,
1700            "failure": {
1701                "message": message.into(),
1702                "type": failure_type
1703            }
1704        });
1705        let path = format!("/worker/workflow-tasks/{task_id}/fail");
1706        self.request_json(
1707            reqwest::Method::POST,
1708            &path,
1709            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1710            Some(&body),
1711        )
1712        .await
1713    }
1714
1715    pub async fn poll_activity_task(
1716        &self,
1717        worker_id: &str,
1718        task_queue: &str,
1719        timeout: Duration,
1720    ) -> Result<Option<ActivityTask>> {
1721        Ok(self
1722            .poll_activity_task_response(worker_id, task_queue, timeout)
1723            .await?
1724            .task)
1725    }
1726
1727    /// Poll an activity task while preserving server stop and drain metadata.
1728    pub async fn poll_activity_task_response(
1729        &self,
1730        worker_id: &str,
1731        task_queue: &str,
1732        timeout: Duration,
1733    ) -> Result<PollActivityTaskResponse> {
1734        let body = json!({
1735            "worker_id": worker_id,
1736            "task_queue": task_queue,
1737            "timeout_seconds": long_poll_timeout_seconds(timeout),
1738        });
1739        let data: PollActivityTaskResponse = worker_poll_response(
1740            self.request_json_with_timeout(
1741                reqwest::Method::POST,
1742                "/worker/activity-tasks/poll",
1743                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1744                Some(&body),
1745                timeout + Duration::from_secs(5),
1746            )
1747            .await,
1748        )?;
1749        Ok(data)
1750    }
1751
1752    pub async fn complete_activity_task(
1753        &self,
1754        task_id: &str,
1755        activity_attempt_id: &str,
1756        lease_owner: &str,
1757        result: Value,
1758        codec: &str,
1759    ) -> Result<Value> {
1760        let result = encode_value_envelope(&result, codec)?;
1761        let body = json!({
1762            "activity_attempt_id": activity_attempt_id,
1763            "lease_owner": lease_owner,
1764            "result": result
1765        });
1766        let path = format!("/worker/activity-tasks/{task_id}/complete");
1767        activity_task_response(
1768            self.request_json(
1769                reqwest::Method::POST,
1770                &path,
1771                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1772                Some(&body),
1773            )
1774            .await,
1775            "complete",
1776            task_id,
1777            activity_attempt_id,
1778        )
1779    }
1780
1781    pub async fn fail_activity_task(
1782        &self,
1783        task_id: &str,
1784        activity_attempt_id: &str,
1785        lease_owner: &str,
1786        message: impl Into<String>,
1787        non_retryable: bool,
1788    ) -> Result<Value> {
1789        let body = json!({
1790            "activity_attempt_id": activity_attempt_id,
1791            "lease_owner": lease_owner,
1792            "failure": {
1793                "message": message.into(),
1794                "type": "RustActivityFailure",
1795                "non_retryable": non_retryable
1796            }
1797        });
1798        let path = format!("/worker/activity-tasks/{task_id}/fail");
1799        activity_task_response(
1800            self.request_json(
1801                reqwest::Method::POST,
1802                &path,
1803                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1804                Some(&body),
1805            )
1806            .await,
1807            "fail",
1808            task_id,
1809            activity_attempt_id,
1810        )
1811    }
1812
1813    pub async fn heartbeat_activity_task(
1814        &self,
1815        task_id: &str,
1816        activity_attempt_id: &str,
1817        lease_owner: &str,
1818        details: Value,
1819    ) -> Result<ActivityHeartbeatResponse> {
1820        let body = json!({
1821            "activity_attempt_id": activity_attempt_id,
1822            "lease_owner": lease_owner,
1823            "details": details
1824        });
1825        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
1826        activity_task_response(
1827            self.request_json(
1828                reqwest::Method::POST,
1829                &path,
1830                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1831                Some(&body),
1832            )
1833            .await,
1834            "heartbeat",
1835            task_id,
1836            activity_attempt_id,
1837        )
1838    }
1839
1840    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
1841        &self,
1842        method: reqwest::Method,
1843        path: &str,
1844        protocol: RequestProtocol,
1845        body: Option<&B>,
1846    ) -> Result<T> {
1847        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
1848            .await
1849    }
1850
1851    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
1852        &self,
1853        method: reqwest::Method,
1854        path: &str,
1855        protocol: RequestProtocol,
1856        body: Option<&B>,
1857        timeout: Duration,
1858    ) -> Result<T> {
1859        let mut request = self
1860            .http
1861            .request(method, format!("{}/api{}", self.base_url, path))
1862            .timeout(timeout)
1863            .header(reqwest::header::ACCEPT, "application/json")
1864            .header(reqwest::header::CONTENT_TYPE, "application/json")
1865            .header("X-Namespace", &self.namespace);
1866
1867        match protocol {
1868            RequestProtocol::Worker(version) => {
1869                request = request.header("X-Durable-Workflow-Protocol-Version", version);
1870            }
1871            RequestProtocol::ControlPlane => {
1872                request = request.header(
1873                    "X-Durable-Workflow-Control-Plane-Version",
1874                    CONTROL_PLANE_VERSION,
1875                );
1876            }
1877        }
1878
1879        if let Some(token) = self.auth_token(protocol.is_worker()) {
1880            request = request.bearer_auth(token);
1881        }
1882
1883        if let Some(body) = body {
1884            request = request.json(body);
1885        }
1886
1887        let response = request.send().await?;
1888        let status = response.status();
1889        let bytes = response.bytes().await?;
1890
1891        if !status.is_success() {
1892            let body = String::from_utf8_lossy(&bytes).to_string();
1893            if let Some(protocol) = protocol_failure(status, &body) {
1894                return Err(Error::Protocol(protocol));
1895            }
1896            return Err(Error::Http { status, body });
1897        }
1898
1899        if bytes.is_empty() {
1900            return Ok(serde_json::from_value(Value::Null)?);
1901        }
1902
1903        Ok(serde_json::from_slice(&bytes)?)
1904    }
1905
1906    fn auth_token(&self, worker: bool) -> Option<&str> {
1907        if worker {
1908            self.worker_token
1909                .as_deref()
1910                .or(self.token.as_deref())
1911                .or(self.control_token.as_deref())
1912        } else {
1913            self.control_token
1914                .as_deref()
1915                .or(self.token.as_deref())
1916                .or(self.worker_token.as_deref())
1917        }
1918    }
1919}
1920
1921fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
1922    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1923    let reason = body
1924        .get("reason")
1925        .and_then(Value::as_str)
1926        .unwrap_or("query_rejected")
1927        .to_string();
1928    let message = body
1929        .get("message")
1930        .or_else(|| body.get("error"))
1931        .and_then(Value::as_str)
1932        .unwrap_or("workflow query was rejected")
1933        .to_string();
1934
1935    QueryFailure {
1936        status: status.as_u16(),
1937        reason,
1938        message,
1939        body,
1940    }
1941}
1942
1943fn workflow_command_result(
1944    command: WorkflowCommandKind,
1945    data: Value,
1946    workflow_id: &str,
1947    run_id: Option<&str>,
1948) -> WorkflowCommandResult {
1949    WorkflowCommandResult {
1950        command,
1951        workflow_id: data
1952            .get("workflow_id")
1953            .and_then(Value::as_str)
1954            .unwrap_or(workflow_id)
1955            .to_string(),
1956        run_id: data
1957            .get("run_id")
1958            .and_then(Value::as_str)
1959            .or(run_id)
1960            .map(str::to_string),
1961        outcome: data
1962            .get("outcome")
1963            .and_then(Value::as_str)
1964            .map(str::to_string),
1965        reason: data
1966            .get("reason")
1967            .and_then(Value::as_str)
1968            .map(str::to_string),
1969        command_status: data
1970            .get("command_status")
1971            .and_then(Value::as_str)
1972            .map(str::to_string),
1973        raw: data,
1974    }
1975}
1976
1977fn workflow_command_rejection(
1978    command: WorkflowCommandKind,
1979    status: reqwest::StatusCode,
1980    raw_body: String,
1981    workflow_id: &str,
1982    run_id: Option<&str>,
1983) -> WorkflowCommandRejection {
1984    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1985    WorkflowCommandRejection {
1986        command,
1987        status: status.as_u16(),
1988        reason: body
1989            .get("reason")
1990            .and_then(Value::as_str)
1991            .unwrap_or("workflow_command_rejected")
1992            .to_string(),
1993        message: body
1994            .get("message")
1995            .or_else(|| body.get("error"))
1996            .and_then(Value::as_str)
1997            .unwrap_or("workflow lifecycle command was rejected")
1998            .to_string(),
1999        workflow_id: body
2000            .get("workflow_id")
2001            .and_then(Value::as_str)
2002            .unwrap_or(workflow_id)
2003            .to_string(),
2004        run_id: body
2005            .get("run_id")
2006            .and_then(Value::as_str)
2007            .or(run_id)
2008            .map(str::to_string),
2009        target_scope: body
2010            .get("target_scope")
2011            .and_then(Value::as_str)
2012            .map(str::to_string),
2013        body,
2014    }
2015}
2016
2017fn query_task_response(response: Result<Value>) -> Result<Value> {
2018    match response {
2019        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2020        response => response,
2021    }
2022}
2023
2024fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2025    match response {
2026        Err(Error::Http { status, body })
2027            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2028        {
2029            Ok(serde_json::from_str(&body)?)
2030        }
2031        response => response,
2032    }
2033}
2034
2035fn worker_poll_body_is_stop(body: &str) -> bool {
2036    serde_json::from_str::<Value>(body)
2037        .ok()
2038        .is_some_and(|body| {
2039            worker_poll_is_stop(
2040                body.get("poll_status").and_then(Value::as_str),
2041                body.get("reason").and_then(Value::as_str),
2042            )
2043        })
2044}
2045
2046fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2047    matches!(poll_status, Some("draining" | "stopped"))
2048        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2049}
2050
2051fn query_task_rejection_is_final(error: &Error) -> bool {
2052    matches!(
2053        error,
2054        Error::QueryFailed(failure)
2055            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2056    )
2057}
2058
2059fn activity_task_response<T>(
2060    response: Result<T>,
2061    operation: &str,
2062    task_id: &str,
2063    activity_attempt_id: &str,
2064) -> Result<T> {
2065    match response {
2066        Err(Error::Http { status, body }) => {
2067            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2068            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2069                operation: operation.to_string(),
2070                status: status.as_u16(),
2071                reason: body
2072                    .get("reason")
2073                    .and_then(Value::as_str)
2074                    .unwrap_or("activity_task_rejected")
2075                    .to_string(),
2076                task_id: body
2077                    .get("task_id")
2078                    .and_then(Value::as_str)
2079                    .unwrap_or(task_id)
2080                    .to_string(),
2081                activity_attempt_id: body
2082                    .get("activity_attempt_id")
2083                    .and_then(Value::as_str)
2084                    .unwrap_or(activity_attempt_id)
2085                    .to_string(),
2086                cancel_requested: body
2087                    .get("cancel_requested")
2088                    .and_then(Value::as_bool)
2089                    .unwrap_or(false),
2090                can_continue: body.get("can_continue").and_then(Value::as_bool),
2091                run_closed_reason: body
2092                    .get("run_closed_reason")
2093                    .and_then(Value::as_str)
2094                    .map(str::to_string),
2095                body,
2096            }))
2097        }
2098        response => response,
2099    }
2100}
2101
2102fn activity_task_rejection_is_final(error: &Error) -> bool {
2103    matches!(
2104        error,
2105        Error::ActivityTaskRejected(rejection)
2106            if matches!(
2107                rejection.reason.as_str(),
2108                "run_cancelled"
2109                    | "run_terminated"
2110                    | "attempt_closed"
2111                    | "stale_attempt"
2112                    | "activity_cancelled"
2113                    | "task_cancelled"
2114                    | "run_closed"
2115                    | "activity_not_running"
2116                    | "attempt_not_found"
2117            )
2118    )
2119}
2120
2121fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2122    let body: Value = serde_json::from_str(raw_body).ok()?;
2123    let reason = body.get("reason")?.as_str()?;
2124    if !matches!(
2125        reason,
2126        "missing_protocol_version"
2127            | "unsupported_protocol_version"
2128            | "missing_control_plane_version"
2129            | "unsupported_control_plane_version"
2130    ) {
2131        return None;
2132    }
2133
2134    Some(ProtocolFailure {
2135        status: status.as_u16(),
2136        reason: reason.to_string(),
2137        message: body
2138            .get("message")
2139            .or_else(|| body.get("error"))
2140            .and_then(Value::as_str)
2141            .unwrap_or("protocol version rejected")
2142            .to_string(),
2143        supported_version: body
2144            .get("supported_version")
2145            .and_then(Value::as_str)
2146            .map(str::to_string),
2147        requested_version: body
2148            .get("requested_version")
2149            .and_then(Value::as_str)
2150            .map(str::to_string),
2151        body,
2152    })
2153}
2154
2155fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2156    timeout
2157        .as_secs()
2158        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2159        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2160}
2161
2162fn worker_operation_is_retryable(error: &Error) -> bool {
2163    match error {
2164        Error::Transport(error) => {
2165            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2166        }
2167        Error::Http { status, .. } => {
2168            matches!(
2169                *status,
2170                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2171            ) || status.is_server_error()
2172        }
2173        _ => false,
2174    }
2175}
2176
2177fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
2178    let exponent = retry.saturating_sub(1).min(31) as u32;
2179    policy
2180        .initial_backoff
2181        .saturating_mul(1_u32 << exponent)
2182        .min(policy.max_backoff)
2183}
2184
2185#[derive(Debug)]
2186pub struct ClientBuilder {
2187    base_url: String,
2188    token: Option<String>,
2189    control_token: Option<String>,
2190    worker_token: Option<String>,
2191    namespace: String,
2192    timeout: Duration,
2193}
2194
2195impl ClientBuilder {
2196    pub fn token(mut self, token: Option<String>) -> Self {
2197        self.token = token;
2198        self
2199    }
2200
2201    pub fn control_token(mut self, token: Option<String>) -> Self {
2202        self.control_token = token;
2203        self
2204    }
2205
2206    pub fn worker_token(mut self, token: Option<String>) -> Self {
2207        self.worker_token = token;
2208        self
2209    }
2210
2211    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
2212        self.namespace = namespace.into();
2213        self
2214    }
2215
2216    pub fn timeout(mut self, timeout: Duration) -> Self {
2217        self.timeout = timeout;
2218        self
2219    }
2220
2221    pub fn build(self) -> Result<Client> {
2222        Ok(Client {
2223            http: reqwest::Client::builder().timeout(self.timeout).build()?,
2224            base_url: self.base_url.trim_end_matches('/').to_string(),
2225            token: self.token,
2226            control_token: self.control_token,
2227            worker_token: self.worker_token,
2228            namespace: self.namespace,
2229        })
2230    }
2231}
2232
2233#[derive(Clone, Debug)]
2234pub struct WorkflowHandle {
2235    client: Client,
2236    pub workflow_id: String,
2237    pub run_id: Option<String>,
2238    pub workflow_type: String,
2239}
2240
2241impl WorkflowHandle {
2242    pub async fn describe(&self) -> Result<WorkflowDescription> {
2243        self.client.describe_workflow(&self.workflow_id).await
2244    }
2245
2246    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
2247        self.client
2248            .signal_workflow(&self.workflow_id, signal_name, input)
2249            .await
2250    }
2251
2252    /// Request cooperative cancellation of whichever run is current.
2253    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
2254        self.client
2255            .cancel_workflow(&self.workflow_id, options)
2256            .await
2257    }
2258
2259    /// Request cancellation only if this handle's selected run is still current.
2260    pub async fn cancel_selected_run(
2261        &self,
2262        options: WorkflowCommandOptions,
2263    ) -> Result<WorkflowCommandResult> {
2264        let run_id = self.run_id.as_deref().ok_or_else(|| {
2265            Error::Codec("run_id is required for selected-run cancellation".to_string())
2266        })?;
2267        self.client
2268            .cancel_workflow_run(&self.workflow_id, run_id, options)
2269            .await
2270    }
2271
2272    /// Forcefully terminate whichever run is current.
2273    pub async fn terminate(
2274        &self,
2275        options: WorkflowCommandOptions,
2276    ) -> Result<WorkflowCommandResult> {
2277        self.client
2278            .terminate_workflow(&self.workflow_id, options)
2279            .await
2280    }
2281
2282    /// Terminate only if this handle's selected run is still current.
2283    pub async fn terminate_selected_run(
2284        &self,
2285        options: WorkflowCommandOptions,
2286    ) -> Result<WorkflowCommandResult> {
2287        let run_id = self.run_id.as_deref().ok_or_else(|| {
2288            Error::Codec("run_id is required for selected-run termination".to_string())
2289        })?;
2290        self.client
2291            .terminate_workflow_run(&self.workflow_id, run_id, options)
2292            .await
2293    }
2294
2295    /// Execute a named, read-only query against this workflow.
2296    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
2297        self.client
2298            .query_workflow(&self.workflow_id, query_name, input)
2299            .await
2300    }
2301
2302    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
2303        let started = Instant::now();
2304
2305        loop {
2306            let description = match self.run_id.as_deref() {
2307                Some(run_id) => {
2308                    self.client
2309                        .describe_workflow_run(&self.workflow_id, run_id)
2310                        .await?
2311                }
2312                None => self.describe().await?,
2313            };
2314            if description.is_completed() {
2315                return Ok(description.output.unwrap_or(Value::Null));
2316            }
2317
2318            if description.is_terminal() {
2319                let outcome = workflow_terminal_outcome(
2320                    &description,
2321                    &self.workflow_id,
2322                    self.run_id.as_deref(),
2323                );
2324                return Err(match outcome.kind {
2325                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
2326                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
2327                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
2328                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
2329                });
2330            }
2331
2332            if started.elapsed() >= options.timeout {
2333                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
2334                    kind: WorkflowTerminalKind::TimedOut,
2335                    workflow_id: description
2336                        .workflow_id
2337                        .clone()
2338                        .unwrap_or_else(|| self.workflow_id.clone()),
2339                    run_id: description.run_id.clone().or_else(|| self.run_id.clone()),
2340                    reason: "result_wait_timeout".to_string(),
2341                    failure_category: Some("client_timeout".to_string()),
2342                    failure_id: None,
2343                    exception_type: None,
2344                    exception_class: None,
2345                    non_retryable: None,
2346                    message: Some(format!(
2347                        "workflow result was not terminal within {:?}",
2348                        options.timeout
2349                    )),
2350                    exception: None,
2351                    raw: description.raw_value(),
2352                }));
2353            }
2354
2355            tokio::time::sleep(options.poll_interval).await;
2356        }
2357    }
2358}
2359
2360#[derive(Clone, Copy, Debug)]
2361pub struct WorkflowResultOptions {
2362    pub poll_interval: Duration,
2363    pub timeout: Duration,
2364}
2365
2366impl Default for WorkflowResultOptions {
2367    fn default() -> Self {
2368        Self {
2369            poll_interval: Duration::from_millis(500),
2370            timeout: Duration::from_secs(30),
2371        }
2372    }
2373}
2374
2375#[derive(Clone, Debug, Deserialize)]
2376pub struct WorkflowDescription {
2377    pub workflow_id: Option<String>,
2378    pub run_id: Option<String>,
2379    pub workflow_type: Option<String>,
2380    pub status: Option<String>,
2381    #[serde(default)]
2382    pub closed_reason: Option<String>,
2383    #[serde(default)]
2384    pub error: Option<String>,
2385    #[serde(default)]
2386    pub failure: Option<Value>,
2387    #[serde(default)]
2388    pub exception: Option<Value>,
2389    #[serde(default)]
2390    pub failures: Vec<Value>,
2391    #[serde(default)]
2392    pub output: Option<Value>,
2393    #[serde(default)]
2394    pub output_envelope: Option<Value>,
2395    #[serde(flatten)]
2396    pub raw: HashMap<String, Value>,
2397}
2398
2399impl WorkflowDescription {
2400    pub fn is_completed(&self) -> bool {
2401        matches!(self.status.as_deref(), Some("completed" | "Completed"))
2402    }
2403
2404    pub fn is_terminal(&self) -> bool {
2405        matches!(
2406            self.status.as_deref(),
2407            Some(
2408                "completed"
2409                    | "Completed"
2410                    | "failed"
2411                    | "Failed"
2412                    | "cancelled"
2413                    | "Cancelled"
2414                    | "terminated"
2415                    | "Terminated"
2416                    | "timed_out"
2417                    | "TimedOut",
2418            )
2419        )
2420    }
2421
2422    fn decode_payloads(&mut self) -> Result<()> {
2423        if let Some(envelope) = &self.output_envelope {
2424            self.output = Some(decode_wire_value(envelope, DEFAULT_CODEC)?);
2425        }
2426
2427        Ok(())
2428    }
2429
2430    fn raw_value(&self) -> Value {
2431        let mut data = self.raw.clone();
2432        data.insert(
2433            "workflow_id".to_string(),
2434            self.workflow_id
2435                .clone()
2436                .map(Value::String)
2437                .unwrap_or(Value::Null),
2438        );
2439        data.insert(
2440            "run_id".to_string(),
2441            self.run_id
2442                .clone()
2443                .map(Value::String)
2444                .unwrap_or(Value::Null),
2445        );
2446        data.insert(
2447            "workflow_type".to_string(),
2448            self.workflow_type
2449                .clone()
2450                .map(Value::String)
2451                .unwrap_or(Value::Null),
2452        );
2453        data.insert(
2454            "status".to_string(),
2455            self.status
2456                .clone()
2457                .map(Value::String)
2458                .unwrap_or(Value::Null),
2459        );
2460        data.insert(
2461            "closed_reason".to_string(),
2462            self.closed_reason
2463                .clone()
2464                .map(Value::String)
2465                .unwrap_or(Value::Null),
2466        );
2467        if let Some(failure) = &self.failure {
2468            data.insert("failure".to_string(), failure.clone());
2469        }
2470        if let Some(exception) = &self.exception {
2471            data.insert("exception".to_string(), exception.clone());
2472        }
2473        Value::Object(data.into_iter().collect())
2474    }
2475}
2476
2477fn workflow_terminal_outcome(
2478    description: &WorkflowDescription,
2479    workflow_id: &str,
2480    run_id: Option<&str>,
2481) -> WorkflowTerminalOutcome {
2482    let terminal_kind = description
2483        .closed_reason
2484        .as_deref()
2485        .or(description.status.as_deref())
2486        .unwrap_or("failed")
2487        .to_ascii_lowercase();
2488    let kind = match terminal_kind.as_str() {
2489        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
2490        "terminated" => WorkflowTerminalKind::Terminated,
2491        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
2492        _ => WorkflowTerminalKind::Failed,
2493    };
2494    let default_reason = match kind {
2495        WorkflowTerminalKind::Failed => "workflow_failed",
2496        WorkflowTerminalKind::Cancelled => "cancelled",
2497        WorkflowTerminalKind::Terminated => "terminated",
2498        WorkflowTerminalKind::TimedOut => "timed_out",
2499    };
2500    let failure = description
2501        .failure
2502        .as_ref()
2503        .filter(|value| value.is_object());
2504    let nested_failure = failure
2505        .and_then(|value| value.get("failures"))
2506        .and_then(Value::as_array)
2507        .and_then(|failures| failures.last())
2508        .or_else(|| description.failures.last());
2509    let exception = description
2510        .exception
2511        .clone()
2512        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
2513        .or_else(|| {
2514            nested_failure
2515                .and_then(|value| value.get("exception_payload"))
2516                .cloned()
2517        });
2518    let string_field = |name: &str| {
2519        failure
2520            .and_then(|value| value.get(name))
2521            .and_then(Value::as_str)
2522            .or_else(|| {
2523                nested_failure
2524                    .and_then(|value| value.get(name))
2525                    .and_then(Value::as_str)
2526            })
2527            .map(str::to_string)
2528    };
2529    let exception_field = |name: &str| {
2530        exception
2531            .as_ref()
2532            .and_then(|value| value.get(name))
2533            .and_then(Value::as_str)
2534            .map(str::to_string)
2535    };
2536    let message = description
2537        .error
2538        .clone()
2539        .or_else(|| string_field("message"))
2540        .or_else(|| exception_field("message"));
2541    let reason = description
2542        .raw
2543        .get("reason")
2544        .and_then(Value::as_str)
2545        .map(str::to_string)
2546        .or_else(|| {
2547            failure
2548                .and_then(|value| value.get("reason"))
2549                .and_then(Value::as_str)
2550                .map(str::to_string)
2551        })
2552        .or_else(|| description.closed_reason.clone())
2553        .unwrap_or_else(|| default_reason.to_string());
2554    let failure_id = string_field("failure_id").or_else(|| {
2555        nested_failure
2556            .and_then(|value| value.get("id"))
2557            .and_then(Value::as_str)
2558            .map(str::to_string)
2559    });
2560
2561    WorkflowTerminalOutcome {
2562        kind,
2563        workflow_id: description
2564            .workflow_id
2565            .clone()
2566            .unwrap_or_else(|| workflow_id.to_string()),
2567        run_id: description
2568            .run_id
2569            .clone()
2570            .or_else(|| run_id.map(str::to_string)),
2571        reason,
2572        failure_category: string_field("failure_category")
2573            .or_else(|| Some(default_reason.to_string())),
2574        failure_id,
2575        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
2576        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
2577        non_retryable: failure
2578            .and_then(|value| value.get("non_retryable"))
2579            .and_then(Value::as_bool)
2580            .or_else(|| {
2581                nested_failure
2582                    .and_then(|value| value.get("non_retryable"))
2583                    .and_then(Value::as_bool)
2584            }),
2585        message,
2586        exception,
2587        raw: description.raw_value(),
2588    }
2589}
2590
2591#[derive(Clone, Debug, Deserialize)]
2592pub struct RegisterWorkerResponse {
2593    pub worker_id: String,
2594    pub registered: bool,
2595    #[serde(default)]
2596    pub heartbeat_interval_seconds: Option<u64>,
2597    #[serde(default)]
2598    pub protocol_version: Option<String>,
2599    #[serde(default)]
2600    pub server_capabilities: Option<Value>,
2601}
2602
2603#[derive(Clone, Debug, Deserialize)]
2604pub struct PollWorkflowTaskResponse {
2605    #[serde(default)]
2606    pub task: Option<WorkflowTask>,
2607    #[serde(default)]
2608    pub poll_status: Option<String>,
2609    #[serde(default)]
2610    pub reason: Option<String>,
2611    #[serde(default)]
2612    pub protocol_version: Option<String>,
2613    #[serde(default)]
2614    pub server_capabilities: Option<Value>,
2615}
2616
2617impl PollWorkflowTaskResponse {
2618    /// Classify this response without parsing server display text.
2619    pub fn outcome(&self) -> WorkerPollOutcome {
2620        worker_poll_outcome(
2621            self.task.is_some(),
2622            self.poll_status.as_deref(),
2623            self.reason.as_deref(),
2624        )
2625    }
2626}
2627
2628#[derive(Clone, Debug, Deserialize)]
2629pub struct PollActivityTaskResponse {
2630    #[serde(default)]
2631    pub task: Option<ActivityTask>,
2632    #[serde(default)]
2633    pub poll_status: Option<String>,
2634    #[serde(default)]
2635    pub reason: Option<String>,
2636}
2637
2638impl PollActivityTaskResponse {
2639    /// Classify this response without parsing server display text.
2640    pub fn outcome(&self) -> WorkerPollOutcome {
2641        worker_poll_outcome(
2642            self.task.is_some(),
2643            self.poll_status.as_deref(),
2644            self.reason.as_deref(),
2645        )
2646    }
2647}
2648
2649#[derive(Clone, Debug, Deserialize)]
2650pub struct PollQueryTaskResponse {
2651    #[serde(default)]
2652    pub task: Option<QueryTask>,
2653    #[serde(default)]
2654    pub poll_status: Option<String>,
2655    #[serde(default)]
2656    pub reason: Option<String>,
2657}
2658
2659impl PollQueryTaskResponse {
2660    /// Classify this response without parsing server display text.
2661    pub fn outcome(&self) -> WorkerPollOutcome {
2662        worker_poll_outcome(
2663            self.task.is_some(),
2664            self.poll_status.as_deref(),
2665            self.reason.as_deref(),
2666        )
2667    }
2668}
2669
2670/// Stable classification for worker poll responses.
2671#[derive(Clone, Debug, PartialEq, Eq)]
2672pub enum WorkerPollOutcome {
2673    /// A task was leased and is available on the response.
2674    Task,
2675    /// No task was leased, but the worker should continue polling.
2676    Idle {
2677        poll_status: Option<String>,
2678        reason: Option<String>,
2679    },
2680    /// The server asked this worker to stop claiming new work.
2681    Stop {
2682        poll_status: Option<String>,
2683        reason: Option<String>,
2684    },
2685}
2686
2687impl WorkerPollOutcome {
2688    pub fn should_stop(&self) -> bool {
2689        matches!(self, Self::Stop { .. })
2690    }
2691}
2692
2693fn worker_poll_outcome(
2694    has_task: bool,
2695    poll_status: Option<&str>,
2696    reason: Option<&str>,
2697) -> WorkerPollOutcome {
2698    if worker_poll_is_stop(poll_status, reason) {
2699        return WorkerPollOutcome::Stop {
2700            poll_status: poll_status.map(str::to_string),
2701            reason: reason.map(str::to_string),
2702        };
2703    }
2704
2705    if has_task {
2706        WorkerPollOutcome::Task
2707    } else {
2708        WorkerPollOutcome::Idle {
2709            poll_status: poll_status.map(str::to_string),
2710            reason: reason.map(str::to_string),
2711        }
2712    }
2713}
2714
2715/// An ephemeral server-routed query task.
2716#[derive(Clone, Debug, Deserialize)]
2717pub struct QueryTask {
2718    pub query_task_id: String,
2719    #[serde(default = "default_workflow_task_attempt")]
2720    pub query_task_attempt: u64,
2721    #[serde(default)]
2722    pub lease_owner: Option<String>,
2723    #[serde(default)]
2724    pub workflow_id: Option<String>,
2725    #[serde(default)]
2726    pub run_id: Option<String>,
2727    pub workflow_type: String,
2728    pub query_name: String,
2729    #[serde(default = "default_payload_codec")]
2730    pub payload_codec: String,
2731    #[serde(default)]
2732    pub workflow_arguments: Option<Value>,
2733    #[serde(default)]
2734    pub query_arguments: Option<Value>,
2735    #[serde(default)]
2736    pub history_events: Vec<HistoryEvent>,
2737    #[serde(default)]
2738    pub history_export: Option<Value>,
2739    #[serde(default)]
2740    pub run_status: Option<String>,
2741}
2742
2743#[derive(Clone, Debug, Deserialize)]
2744pub struct WorkflowTask {
2745    pub task_id: String,
2746    #[serde(default)]
2747    pub workflow_id: Option<String>,
2748    #[serde(default)]
2749    pub run_id: Option<String>,
2750    pub workflow_type: String,
2751    #[serde(default = "default_payload_codec")]
2752    pub payload_codec: String,
2753    #[serde(default)]
2754    pub arguments: Option<Value>,
2755    #[serde(default)]
2756    pub history_events: Vec<HistoryEvent>,
2757    #[serde(default)]
2758    pub total_history_events: Option<u64>,
2759    #[serde(default)]
2760    pub next_history_page_token: Option<String>,
2761    #[serde(default = "default_workflow_task_attempt")]
2762    pub workflow_task_attempt: u64,
2763    #[serde(default)]
2764    pub workflow_signal_id: Option<String>,
2765    #[serde(default)]
2766    pub signal_name: Option<String>,
2767    #[serde(default)]
2768    pub signal_arguments: Option<Value>,
2769    #[serde(default)]
2770    pub lease_owner: Option<String>,
2771}
2772
2773impl WorkflowTask {
2774    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
2775        self.history_events.extend(page.history_events);
2776
2777        if page.total_history_events.is_some() {
2778            self.total_history_events = page.total_history_events;
2779        }
2780
2781        self.next_history_page_token = page
2782            .next_history_page_token
2783            .filter(|token| !token.is_empty());
2784    }
2785}
2786
2787#[derive(Clone, Debug, Deserialize)]
2788struct WorkflowTaskHistoryPage {
2789    #[serde(default)]
2790    history_events: Vec<HistoryEvent>,
2791    #[serde(default)]
2792    total_history_events: Option<u64>,
2793    #[serde(default)]
2794    next_history_page_token: Option<String>,
2795}
2796
2797#[derive(Clone, Debug, Deserialize)]
2798pub struct ActivityTask {
2799    pub task_id: String,
2800    #[serde(default)]
2801    pub activity_attempt_id: Option<String>,
2802    #[serde(default)]
2803    pub attempt_id: Option<String>,
2804    pub activity_type: String,
2805    #[serde(default = "default_payload_codec")]
2806    pub payload_codec: String,
2807    #[serde(default)]
2808    pub arguments: Option<Value>,
2809    #[serde(default = "default_attempt_number")]
2810    pub attempt_number: u64,
2811    #[serde(default)]
2812    pub lease_owner: Option<String>,
2813}
2814
2815#[derive(Clone, Debug, Deserialize)]
2816pub struct HistoryEvent {
2817    #[serde(alias = "type")]
2818    pub event_type: String,
2819    #[serde(default)]
2820    pub payload: Value,
2821    #[serde(flatten)]
2822    pub raw: HashMap<String, Value>,
2823}
2824
2825/// One decoded signal in the committed workflow-history snapshot.
2826#[derive(Clone, Debug, PartialEq)]
2827pub struct QuerySignal {
2828    pub id: Option<String>,
2829    pub name: String,
2830    pub arguments: Vec<Value>,
2831    pub workflow_sequence: Option<u64>,
2832}
2833
2834/// Immutable state supplied to a registered query handler.
2835///
2836/// This context intentionally exposes no activity, signal-wait, or command
2837/// APIs. Query handlers inspect committed history and return a value; query
2838/// completion does not append an event or advance deterministic execution.
2839#[derive(Clone, Debug)]
2840pub struct QueryContext {
2841    pub workflow_id: Option<String>,
2842    pub run_id: Option<String>,
2843    pub workflow_type: String,
2844    pub run_status: Option<String>,
2845    workflow_input: Value,
2846    history_events: Arc<Vec<HistoryEvent>>,
2847    signal_events: Arc<Vec<QuerySignal>>,
2848}
2849
2850impl QueryContext {
2851    /// The normalized argument list used to start the workflow.
2852    pub fn workflow_input(&self) -> &Value {
2853        &self.workflow_input
2854    }
2855
2856    /// The immutable committed history used for this query snapshot.
2857    pub fn history_events(&self) -> &[HistoryEvent] {
2858        self.history_events.as_slice()
2859    }
2860
2861    /// All decoded signals in committed workflow order.
2862    pub fn signal_events(&self) -> &[QuerySignal] {
2863        self.signal_events.as_slice()
2864    }
2865
2866    /// Decoded argument lists for each committed signal with `signal_name`.
2867    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
2868        self.signal_events
2869            .iter()
2870            .filter(|signal| signal.name == signal_name)
2871            .map(|signal| signal.arguments.clone())
2872            .collect()
2873    }
2874}
2875
2876#[derive(Clone, Debug, Deserialize)]
2877pub struct ActivityHeartbeatResponse {
2878    #[serde(default)]
2879    pub cancel_requested: bool,
2880    #[serde(default)]
2881    pub heartbeat_recorded: bool,
2882    #[serde(default)]
2883    pub can_continue: Option<bool>,
2884    #[serde(default)]
2885    pub reason: Option<String>,
2886    #[serde(default)]
2887    pub run_closed_reason: Option<String>,
2888    #[serde(default)]
2889    pub run_closed_at: Option<String>,
2890    #[serde(default)]
2891    pub lease_expires_at: Option<String>,
2892    #[serde(default)]
2893    pub last_heartbeat_at: Option<String>,
2894}
2895
2896impl ActivityHeartbeatResponse {
2897    /// Whether the activity should stop instead of attempting completion.
2898    pub fn should_stop(&self) -> bool {
2899        self.cancel_requested || self.can_continue == Some(false)
2900    }
2901}
2902
2903fn default_payload_codec() -> String {
2904    DEFAULT_CODEC.to_string()
2905}
2906
2907fn default_workflow_task_attempt() -> u64 {
2908    1
2909}
2910
2911fn default_attempt_number() -> u64 {
2912    1
2913}
2914
2915type WorkflowFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2916type WorkflowHandler = Arc<dyn Fn(WorkflowContext, Value) -> WorkflowFuture + Send + Sync>;
2917type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
2918type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
2919type ReplayedWorkflowHandler =
2920    Arc<dyn Fn(WorkflowContext, Value) -> ReplayedWorkflowInvocation + Send + Sync>;
2921type ActivityFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2922type ActivityHandler = Arc<dyn Fn(ActivityContext, Value) -> ActivityFuture + Send + Sync>;
2923type QueryFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2924type QueryHandler = Arc<dyn Fn(QueryContext, Value) -> QueryFuture + Send + Sync>;
2925type ReplayedQueryHandler = Arc<
2926    dyn Fn(QueryContext, ErasedWorkflowState, Value) -> std::result::Result<QueryFuture, String>
2927        + Send
2928        + Sync,
2929>;
2930type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
2931
2932struct ReplayedWorkflowInvocation {
2933    future: WorkflowFuture,
2934    snapshot: WorkflowStateSnapshot,
2935}
2936
2937#[derive(Clone)]
2938struct RegisteredWorkflow {
2939    execute: WorkflowHandler,
2940    replay: Option<ReplayedWorkflowHandler>,
2941    state_type: Option<TypeId>,
2942}
2943
2944#[derive(Clone)]
2945enum RegisteredQuery {
2946    Snapshot(QueryHandler),
2947    Replayed {
2948        state_type: TypeId,
2949        handler: ReplayedQueryHandler,
2950    },
2951}
2952
2953#[derive(Clone, Debug)]
2954pub struct WorkerHeartbeatObservation {
2955    pub worker_id: String,
2956    pub task_queue: String,
2957    pub acknowledged_at_unix_millis: u64,
2958    pub acknowledgement: Value,
2959}
2960
2961/// Bounded retry policy for worker poll acquisition and worker heartbeats.
2962///
2963/// Expected empty long polls are normal successful responses. Transport
2964/// failures, HTTP 408/429 responses, and server errors are retried with capped
2965/// exponential backoff. Authentication, protocol, codec, and handler failures
2966/// are never retried by the worker.
2967#[derive(Clone, Copy, Debug)]
2968pub struct WorkerRetryPolicy {
2969    /// Number of retries after the initial request fails.
2970    pub max_retries: usize,
2971    /// Delay before the first retry.
2972    pub initial_backoff: Duration,
2973    /// Maximum delay between retries.
2974    pub max_backoff: Duration,
2975}
2976
2977impl Default for WorkerRetryPolicy {
2978    fn default() -> Self {
2979        Self {
2980            max_retries: 5,
2981            initial_backoff: Duration::from_millis(100),
2982            max_backoff: Duration::from_secs(5),
2983        }
2984    }
2985}
2986
2987#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2988enum ManagedPollOutcome {
2989    Idle,
2990    Handled,
2991    Stop,
2992}
2993
2994#[derive(Clone)]
2995pub struct Worker {
2996    client: Client,
2997    worker_id: String,
2998    task_queue: String,
2999    workflows: HashMap<String, RegisteredWorkflow>,
3000    activities: HashMap<String, ActivityHandler>,
3001    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
3002    max_concurrent_workflow_tasks: usize,
3003    max_concurrent_activity_tasks: usize,
3004    poll_timeout: Duration,
3005    heartbeat_interval: Duration,
3006    retry_policy: WorkerRetryPolicy,
3007    heartbeat_observer: Option<WorkerHeartbeatObserver>,
3008}
3009
3010impl Worker {
3011    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
3012        Self {
3013            client,
3014            worker_id: default_worker_id(),
3015            task_queue: task_queue.into(),
3016            workflows: HashMap::new(),
3017            activities: HashMap::new(),
3018            queries: HashMap::new(),
3019            max_concurrent_workflow_tasks: 10,
3020            max_concurrent_activity_tasks: 10,
3021            poll_timeout: Duration::from_secs(30),
3022            heartbeat_interval: Duration::from_secs(60),
3023            retry_policy: WorkerRetryPolicy::default(),
3024            heartbeat_observer: None,
3025        }
3026    }
3027
3028    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
3029        self.worker_id = worker_id.into();
3030        self
3031    }
3032
3033    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
3034        self.poll_timeout = timeout;
3035        self
3036    }
3037
3038    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
3039        self.heartbeat_interval = interval;
3040        self
3041    }
3042
3043    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
3044    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
3045        self.retry_policy = policy;
3046        self
3047    }
3048
3049    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
3050    where
3051        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
3052    {
3053        self.heartbeat_observer = Some(Arc::new(observer));
3054        self
3055    }
3056
3057    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
3058        self.max_concurrent_workflow_tasks = count.max(1);
3059        self
3060    }
3061
3062    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
3063        self.max_concurrent_activity_tasks = count.max(1);
3064        self
3065    }
3066
3067    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
3068    where
3069        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
3070        Fut: Future<Output = Result<Value>> + Send + 'static,
3071    {
3072        self.workflows.insert(
3073            workflow_type.into(),
3074            RegisteredWorkflow {
3075                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
3076                replay: None,
3077                state_type: None,
3078            },
3079        );
3080    }
3081
3082    /// Register a workflow whose typed instance state can be reconstructed for queries.
3083    ///
3084    /// `state_factory` creates a fresh instance for every normal workflow task and
3085    /// query replay. The workflow handler is the single source of truth for state
3086    /// transitions: it updates [`WorkflowInstance`] after activities and signals
3087    /// resolve. Query replay runs this same handler over committed history and
3088    /// discards any commands it would emit.
3089    pub fn register_replayed_workflow<S, Factory, F, Fut>(
3090        &mut self,
3091        workflow_type: impl Into<String>,
3092        state_factory: Factory,
3093        handler: F,
3094    ) where
3095        S: Clone + Send + Sync + 'static,
3096        Factory: Fn() -> S + Send + Sync + 'static,
3097        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
3098        Fut: Future<Output = Result<Value>> + Send + 'static,
3099    {
3100        let state_factory = Arc::new(state_factory);
3101        let handler = Arc::new(handler);
3102
3103        let execute_factory = Arc::clone(&state_factory);
3104        let execute_handler = Arc::clone(&handler);
3105        let execute = Arc::new(move |ctx: WorkflowContext, input: Value| {
3106            let state = WorkflowInstance::new(execute_factory());
3107            let future = execute_handler(ctx, input, state);
3108            Box::pin(future) as WorkflowFuture
3109        });
3110
3111        let replay = Arc::new(move |ctx: WorkflowContext, input: Value| {
3112            let state = WorkflowInstance::new(state_factory());
3113            let snapshot_state = state.clone();
3114            let snapshot: WorkflowStateSnapshot =
3115                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
3116            let future = handler(ctx, input, state);
3117            ReplayedWorkflowInvocation {
3118                future: Box::pin(future),
3119                snapshot,
3120            }
3121        });
3122
3123        self.workflows.insert(
3124            workflow_type.into(),
3125            RegisteredWorkflow {
3126                execute,
3127                replay: Some(replay),
3128                state_type: Some(TypeId::of::<S>()),
3129            },
3130        );
3131    }
3132
3133    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
3134    where
3135        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
3136        Fut: Future<Output = Result<Value>> + Send + 'static,
3137    {
3138        self.activities.insert(
3139            activity_type.into(),
3140            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
3141        );
3142    }
3143
3144    /// Register a named, read-only query handler for a workflow type.
3145    ///
3146    /// The workflow type must also be registered with [`Worker::register_workflow`]
3147    /// before the worker runs. The handler receives only an immutable committed
3148    /// state snapshot and normalized query arguments.
3149    pub fn register_query<F, Fut>(
3150        &mut self,
3151        workflow_type: impl Into<String>,
3152        query_name: impl Into<String>,
3153        handler: F,
3154    ) where
3155        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
3156        Fut: Future<Output = Result<Value>> + Send + 'static,
3157    {
3158        self.queries
3159            .entry(workflow_type.into())
3160            .or_default()
3161            .insert(
3162                query_name.into(),
3163                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
3164            );
3165    }
3166
3167    /// Register a named query against deterministically replayed instance state.
3168    ///
3169    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
3170    /// same state type `S`. The handler receives an immutable, detached state
3171    /// clone, so successful and failed queries cannot affect workflow execution
3172    /// or the state reconstructed by a later query.
3173    pub fn register_replayed_query<S, F, Fut>(
3174        &mut self,
3175        workflow_type: impl Into<String>,
3176        query_name: impl Into<String>,
3177        handler: F,
3178    ) where
3179        S: Clone + Send + Sync + 'static,
3180        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
3181        Fut: Future<Output = Result<Value>> + Send + 'static,
3182    {
3183        let handler = Arc::new(handler);
3184        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
3185            let state = state.downcast::<S>().map_err(|_| {
3186                "registered query state type does not match the replayed workflow state".to_string()
3187            })?;
3188            Ok(Box::pin(handler(ctx, state, args)))
3189        });
3190
3191        self.queries
3192            .entry(workflow_type.into())
3193            .or_default()
3194            .insert(
3195                query_name.into(),
3196                RegisteredQuery::Replayed {
3197                    state_type: TypeId::of::<S>(),
3198                    handler: erased_handler,
3199                },
3200            );
3201    }
3202
3203    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
3204        self.client
3205            .register_worker_with_capabilities(
3206                &self.worker_id,
3207                &self.task_queue,
3208                self.workflows.keys().cloned().collect(),
3209                self.activities.keys().cloned().collect(),
3210                self.max_concurrent_workflow_tasks,
3211                self.max_concurrent_activity_tasks,
3212                (!self.queries.is_empty())
3213                    .then(|| QUERY_TASKS_CAPABILITY.to_string())
3214                    .into_iter()
3215                    .collect(),
3216            )
3217            .await
3218    }
3219
3220    /// Run until shutdown or a terminal worker error occurs.
3221    ///
3222    /// Empty long-poll expirations do not stop the worker. Retryable poll and
3223    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
3224    /// authentication, protocol, and other non-retryable failures are returned.
3225    pub async fn run(&self) -> Result<()> {
3226        self.run_until(std::future::pending::<()>()).await
3227    }
3228
3229    /// Run until `shutdown` resolves or a terminal worker error occurs.
3230    ///
3231    /// This has the same liveness and terminal-error contract as [`Worker::run`].
3232    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
3233    where
3234        F: Future<Output = ()>,
3235    {
3236        let registration = self.register().await?;
3237        let heartbeat_interval = Duration::from_secs(
3238            registration
3239                .heartbeat_interval_seconds
3240                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
3241        );
3242        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
3243        // from the completion of the preceding attempt, including its bounded
3244        // retries. A fixed-epoch interval can leave an already-due tick queued
3245        // while an acknowledgement is slow, producing a catch-up heartbeat as
3246        // soon as that request completes.
3247        let heartbeat = tokio::time::sleep(Duration::ZERO);
3248        tokio::pin!(heartbeat);
3249        tokio::pin!(shutdown);
3250        let stop = Arc::new(AtomicBool::new(false));
3251        // Poll responses may already have leased server-side work by the time
3252        // they become ready, so each poller owns its responses through
3253        // completion or failure instead of racing raw polls in this select.
3254        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
3255            let worker = self.clone();
3256            let stop = Arc::clone(&stop);
3257            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
3258        });
3259        let mut activity_poller = (!self.activities.is_empty()).then(|| {
3260            let worker = self.clone();
3261            let stop = Arc::clone(&stop);
3262            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
3263        });
3264        let mut query_poller = (!self.queries.is_empty()).then(|| {
3265            let worker = self.clone();
3266            let stop = Arc::clone(&stop);
3267            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
3268        });
3269
3270        loop {
3271            tokio::select! {
3272                _ = &mut shutdown => {
3273                    stop.store(true, Ordering::SeqCst);
3274                    break;
3275                }
3276                _ = &mut heartbeat => {
3277                    let result = self.retry_worker_operation(|| {
3278                        self.client.heartbeat_worker(
3279                            &self.worker_id,
3280                            self.max_concurrent_workflow_tasks,
3281                            self.max_concurrent_activity_tasks,
3282                        )
3283                    }).await;
3284                    heartbeat
3285                        .as_mut()
3286                        .reset(tokio::time::Instant::now() + heartbeat_interval);
3287                    match result {
3288                        Ok(acknowledgement) => {
3289                            if let Some(observer) = &self.heartbeat_observer {
3290                                observer(&WorkerHeartbeatObservation {
3291                                    worker_id: self.worker_id.clone(),
3292                                    task_queue: self.task_queue.clone(),
3293                                    acknowledged_at_unix_millis: SystemTime::now()
3294                                        .duration_since(UNIX_EPOCH)
3295                                        .unwrap_or_default()
3296                                        .as_millis()
3297                                        .min(u64::MAX as u128)
3298                                        as u64,
3299                                    acknowledgement,
3300                                });
3301                            }
3302                        }
3303                        Err(error) => {
3304                            stop.store(true, Ordering::SeqCst);
3305                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
3306                            return Err(error);
3307                        }
3308                    }
3309                }
3310                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
3311                    workflow_poller = None;
3312                    let stopped_by_server = stop.load(Ordering::SeqCst);
3313                    stop.store(true, Ordering::SeqCst);
3314                    let poller_result = optional_poller_result("workflow", result);
3315                    let join_result =
3316                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3317                    poller_result?;
3318                    join_result?;
3319                    if stopped_by_server {
3320                        return Ok(());
3321                    }
3322                    return Err(Error::WorkerLoop(
3323                        "workflow poller stopped unexpectedly".to_string(),
3324                    ));
3325                }
3326                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
3327                    activity_poller = None;
3328                    let stopped_by_server = stop.load(Ordering::SeqCst);
3329                    stop.store(true, Ordering::SeqCst);
3330                    let poller_result = optional_poller_result("activity", result);
3331                    let join_result =
3332                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3333                    poller_result?;
3334                    join_result?;
3335                    if stopped_by_server {
3336                        return Ok(());
3337                    }
3338                    return Err(Error::WorkerLoop(
3339                        "activity poller stopped unexpectedly".to_string(),
3340                    ));
3341                }
3342                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
3343                    query_poller = None;
3344                    let stopped_by_server = stop.load(Ordering::SeqCst);
3345                    stop.store(true, Ordering::SeqCst);
3346                    let poller_result = optional_poller_result("query", result);
3347                    let join_result =
3348                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3349                    poller_result?;
3350                    join_result?;
3351                    if stopped_by_server {
3352                        return Ok(());
3353                    }
3354                    return Err(Error::WorkerLoop(
3355                        "query poller stopped unexpectedly".to_string(),
3356                    ));
3357                }
3358            }
3359        }
3360
3361        join_pollers(
3362            workflow_poller.take(),
3363            activity_poller.take(),
3364            query_poller.take(),
3365        )
3366        .await
3367    }
3368
3369    pub async fn run_once(&self) -> Result<usize> {
3370        let mut handled = 0;
3371        match self.poll_workflow_once().await? {
3372            ManagedPollOutcome::Handled => handled += 1,
3373            ManagedPollOutcome::Stop => return Ok(handled),
3374            ManagedPollOutcome::Idle => {}
3375        }
3376        match self.poll_activity_once().await? {
3377            ManagedPollOutcome::Handled => handled += 1,
3378            ManagedPollOutcome::Stop => return Ok(handled),
3379            ManagedPollOutcome::Idle => {}
3380        }
3381        if !self.queries.is_empty() {
3382            match self.poll_query_once().await? {
3383                ManagedPollOutcome::Handled => handled += 1,
3384                ManagedPollOutcome::Stop => return Ok(handled),
3385                ManagedPollOutcome::Idle => {}
3386            }
3387        }
3388        Ok(handled)
3389    }
3390
3391    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
3392        let response = self
3393            .retry_worker_operation(|| {
3394                self.client.poll_workflow_task_response(
3395                    &self.worker_id,
3396                    &self.task_queue,
3397                    self.poll_timeout,
3398                )
3399            })
3400            .await?;
3401        if response.outcome().should_stop() {
3402            return Ok(ManagedPollOutcome::Stop);
3403        }
3404        let Some(task) = response.task else {
3405            return Ok(ManagedPollOutcome::Idle);
3406        };
3407
3408        let task_id = task.task_id.clone();
3409        let attempt = task.workflow_task_attempt;
3410        let lease_owner = task
3411            .lease_owner
3412            .clone()
3413            .unwrap_or_else(|| self.worker_id.clone());
3414
3415        match self.execute_workflow_task(task) {
3416            Ok(commands) if commands.is_empty() => {
3417                // A replay can consume a recorded pending durable command
3418                // without producing a new command. The standalone protocol
3419                // acknowledges that state through the typed waiting outcome;
3420                // an empty completion is rejected by servers that require at
3421                // least one executable command.
3422                self.client
3423                    .fail_workflow_task_with_type(
3424                        &task_id,
3425                        &lease_owner,
3426                        attempt,
3427                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
3428                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
3429                    )
3430                    .await?;
3431            }
3432            Ok(commands) => {
3433                self.client
3434                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
3435                    .await?;
3436            }
3437            Err(error) => {
3438                self.client
3439                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
3440                    .await?;
3441            }
3442        }
3443
3444        Ok(ManagedPollOutcome::Handled)
3445    }
3446
3447    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3448        while !stop.load(Ordering::SeqCst) {
3449            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
3450                stop.store(true, Ordering::SeqCst);
3451                break;
3452            }
3453        }
3454
3455        Ok(())
3456    }
3457
3458    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
3459        let response = self
3460            .retry_worker_operation(|| {
3461                self.client.poll_activity_task_response(
3462                    &self.worker_id,
3463                    &self.task_queue,
3464                    self.poll_timeout,
3465                )
3466            })
3467            .await?;
3468        if response.outcome().should_stop() {
3469            return Ok(ManagedPollOutcome::Stop);
3470        }
3471        let Some(task) = response.task else {
3472            return Ok(ManagedPollOutcome::Idle);
3473        };
3474
3475        let task_id = task.task_id.clone();
3476        let attempt_id = task
3477            .activity_attempt_id
3478            .clone()
3479            .or(task.attempt_id.clone())
3480            .unwrap_or_default();
3481        let lease_owner = task
3482            .lease_owner
3483            .clone()
3484            .unwrap_or_else(|| self.worker_id.clone());
3485        let codec = task.payload_codec.clone();
3486        let result = self.execute_activity_task(task).await;
3487        match result {
3488            Ok(value) => {
3489                let completion = self
3490                    .client
3491                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
3492                    .await;
3493                if let Err(error) = completion {
3494                    if !activity_task_rejection_is_final(&error) {
3495                        return Err(error);
3496                    }
3497                }
3498            }
3499            Err(error) => {
3500                let failure = self
3501                    .client
3502                    .fail_activity_task(
3503                        &task_id,
3504                        &attempt_id,
3505                        &lease_owner,
3506                        error.to_string(),
3507                        false,
3508                    )
3509                    .await;
3510                if let Err(error) = failure {
3511                    if !activity_task_rejection_is_final(&error) {
3512                        return Err(error);
3513                    }
3514                }
3515            }
3516        }
3517
3518        Ok(ManagedPollOutcome::Handled)
3519    }
3520
3521    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3522        while !stop.load(Ordering::SeqCst) {
3523            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
3524                stop.store(true, Ordering::SeqCst);
3525                break;
3526            }
3527        }
3528
3529        Ok(())
3530    }
3531
3532    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
3533        let response = self
3534            .retry_worker_operation(|| {
3535                self.client.poll_query_task_response(
3536                    &self.worker_id,
3537                    &self.task_queue,
3538                    self.poll_timeout,
3539                )
3540            })
3541            .await?;
3542        if response.outcome().should_stop() {
3543            return Ok(ManagedPollOutcome::Stop);
3544        }
3545        let Some(task) = response.task else {
3546            return Ok(ManagedPollOutcome::Idle);
3547        };
3548
3549        let query_task_id = task.query_task_id.clone();
3550        let attempt = task.query_task_attempt;
3551        let lease_owner = task
3552            .lease_owner
3553            .clone()
3554            .unwrap_or_else(|| self.worker_id.clone());
3555        let codec = task.payload_codec.clone();
3556
3557        match self.execute_query_task(task).await {
3558            Ok(value) => {
3559                let result_envelope = match encode_value_envelope(&value, &codec) {
3560                    Ok(result_envelope) => result_envelope,
3561                    Err(error) => {
3562                        let failure = self
3563                            .client
3564                            .fail_query_task(
3565                                &query_task_id,
3566                                &lease_owner,
3567                                attempt,
3568                                error.to_string(),
3569                                "query_result_encode_failed",
3570                                "QueryResultEncodeFailed",
3571                            )
3572                            .await;
3573                        if let Err(error) = failure {
3574                            if !query_task_rejection_is_final(&error) {
3575                                return Err(error);
3576                            }
3577                        }
3578                        return Ok(ManagedPollOutcome::Handled);
3579                    }
3580                };
3581
3582                if let Err(error) = self
3583                    .client
3584                    .complete_query_task_with_envelope(
3585                        &query_task_id,
3586                        &lease_owner,
3587                        attempt,
3588                        value,
3589                        result_envelope,
3590                    )
3591                    .await
3592                {
3593                    if !query_task_rejection_is_final(&error) {
3594                        return Err(error);
3595                    }
3596                }
3597            }
3598            Err(failure) => {
3599                let result = self
3600                    .client
3601                    .fail_query_task(
3602                        &query_task_id,
3603                        &lease_owner,
3604                        attempt,
3605                        failure.message,
3606                        failure.reason,
3607                        failure.failure_type,
3608                    )
3609                    .await;
3610                if let Err(error) = result {
3611                    if !query_task_rejection_is_final(&error) {
3612                        return Err(error);
3613                    }
3614                }
3615            }
3616        }
3617
3618        Ok(ManagedPollOutcome::Handled)
3619    }
3620
3621    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3622        while !stop.load(Ordering::SeqCst) {
3623            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
3624                stop.store(true, Ordering::SeqCst);
3625                break;
3626            }
3627        }
3628
3629        Ok(())
3630    }
3631
3632    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
3633    where
3634        F: FnMut() -> Fut,
3635        Fut: Future<Output = Result<T>>,
3636    {
3637        let mut retries = 0;
3638
3639        loop {
3640            match operation().await {
3641                Err(error)
3642                    if worker_operation_is_retryable(&error)
3643                        && retries < self.retry_policy.max_retries =>
3644                {
3645                    retries += 1;
3646                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
3647                }
3648                result => return result,
3649            }
3650        }
3651    }
3652
3653    async fn execute_query_task(
3654        &self,
3655        mut task: QueryTask,
3656    ) -> std::result::Result<Value, QueryTaskExecutionFailure> {
3657        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
3658            return Err(QueryTaskExecutionFailure::new(
3659                "query_payload_decode_failed",
3660                format!(
3661                    "cannot decode query payload with unsupported codec {:?}",
3662                    task.payload_codec
3663                ),
3664                "QueryPayloadDecodeFailed",
3665            ));
3666        }
3667
3668        if !self.workflows.contains_key(&task.workflow_type) {
3669            return Err(QueryTaskExecutionFailure::new(
3670                "query_workflow_type_not_registered",
3671                format!("no workflow registered for type {:?}", task.workflow_type),
3672                "WorkflowTypeNotRegistered",
3673            ));
3674        }
3675
3676        let Some(handlers) = self.queries.get(&task.workflow_type) else {
3677            return Err(QueryTaskExecutionFailure::new(
3678                "query_handler_unavailable",
3679                format!(
3680                    "query handlers are unavailable for workflow type {:?}",
3681                    task.workflow_type
3682                ),
3683                "QueryHandlerUnavailable",
3684            ));
3685        };
3686        let Some(query) = handlers.get(&task.query_name) else {
3687            return Err(QueryTaskExecutionFailure::new(
3688                "rejected_unknown_query",
3689                format!("unknown query {:?}", task.query_name),
3690                "QueryFailed",
3691            ));
3692        };
3693
3694        let args = decode_task_arguments(task.query_arguments.as_ref(), &task.payload_codec)
3695            .map_err(|error| {
3696                QueryTaskExecutionFailure::new(
3697                    "query_payload_decode_failed",
3698                    format!("cannot decode query arguments: {error}"),
3699                    "QueryPayloadDecodeFailed",
3700                )
3701            })?;
3702        let workflow_input =
3703            decode_task_arguments(task.workflow_arguments.as_ref(), &task.payload_codec).map_err(
3704                |error| {
3705                    QueryTaskExecutionFailure::new(
3706                        "query_workflow_state_unavailable",
3707                        format!("cannot decode workflow start input: {error}"),
3708                        "QueryWorkflowStateUnavailable",
3709                    )
3710                },
3711            )?;
3712        hydrate_query_history_from_export(&mut task).map_err(|error| {
3713            QueryTaskExecutionFailure::new(
3714                "query_workflow_state_unavailable",
3715                format!("cannot restore query history snapshot: {error}"),
3716                "QueryWorkflowStateUnavailable",
3717            )
3718        })?;
3719        enrich_query_history_from_export(&mut task).map_err(|error| {
3720            QueryTaskExecutionFailure::new(
3721                "query_workflow_state_unavailable",
3722                format!("cannot restore compact query history payloads: {error}"),
3723                "QueryWorkflowStateUnavailable",
3724            )
3725        })?;
3726        let signal_events = query_signal_events(&task).map_err(|error| {
3727            QueryTaskExecutionFailure::new(
3728                "query_workflow_state_unavailable",
3729                format!("cannot decode committed workflow signals: {error}"),
3730                "QueryWorkflowStateUnavailable",
3731            )
3732        })?;
3733        let history_events = Arc::new(std::mem::take(&mut task.history_events));
3734        let context = QueryContext {
3735            workflow_id: task.workflow_id,
3736            run_id: task.run_id,
3737            workflow_type: task.workflow_type.clone(),
3738            run_status: task.run_status,
3739            workflow_input,
3740            history_events: Arc::clone(&history_events),
3741            signal_events: Arc::new(signal_events),
3742        };
3743
3744        let future = match query {
3745            RegisteredQuery::Snapshot(handler) => handler(context, args),
3746            RegisteredQuery::Replayed {
3747                state_type,
3748                handler,
3749            } => {
3750                let workflow = self
3751                    .workflows
3752                    .get(&task.workflow_type)
3753                    .expect("workflow registration was checked above");
3754                if workflow.state_type != Some(*state_type) {
3755                    return Err(QueryTaskExecutionFailure::new(
3756                        "query_workflow_state_unavailable",
3757                        "replayed query state type does not match its workflow registration",
3758                        "QueryWorkflowStateUnavailable",
3759                    ));
3760                }
3761                let replay = workflow.replay.as_ref().ok_or_else(|| {
3762                    QueryTaskExecutionFailure::new(
3763                        "query_workflow_state_unavailable",
3764                        format!(
3765                            "workflow type {:?} is not registered for instance-state replay",
3766                            task.workflow_type
3767                        ),
3768                        "QueryWorkflowStateUnavailable",
3769                    )
3770                })?;
3771                let workflow_state = Arc::new(Mutex::new(
3772                    WorkflowState::new_with_identity(
3773                        history_events.as_ref().clone(),
3774                        context.workflow_id.clone(),
3775                        context.run_id.clone(),
3776                        self.task_queue.clone(),
3777                        task.payload_codec,
3778                        None,
3779                    )
3780                    .map_err(|error| {
3781                        QueryTaskExecutionFailure::new(
3782                            "query_workflow_state_unavailable",
3783                            format!("workflow replay failed before query: {error}"),
3784                            "QueryWorkflowStateUnavailable",
3785                        )
3786                    })?,
3787                ));
3788                let workflow_context = WorkflowContext {
3789                    state: workflow_state,
3790                };
3791                let mut invocation =
3792                    replay(workflow_context.clone(), context.workflow_input.clone());
3793                let mut cx = TaskContext::from_waker(noop_waker_ref());
3794                match invocation.future.as_mut().poll(&mut cx) {
3795                    Poll::Ready(Ok(_)) => {
3796                        workflow_context
3797                            .ensure_history_consumed()
3798                            .map_err(|error| {
3799                                QueryTaskExecutionFailure::new(
3800                                    "query_workflow_state_unavailable",
3801                                    format!("workflow replay failed before query: {error}"),
3802                                    "QueryWorkflowStateUnavailable",
3803                                )
3804                            })?;
3805                    }
3806                    Poll::Ready(Err(error)) => {
3807                        return Err(QueryTaskExecutionFailure::new(
3808                            "query_workflow_state_unavailable",
3809                            format!("workflow replay failed before query: {error}"),
3810                            "QueryWorkflowStateUnavailable",
3811                        ));
3812                    }
3813                    Poll::Pending => {
3814                        let commands = workflow_context.take_commands().map_err(|error| {
3815                            QueryTaskExecutionFailure::new(
3816                                "query_workflow_state_unavailable",
3817                                format!("workflow replay failed before query: {error}"),
3818                                "QueryWorkflowStateUnavailable",
3819                            )
3820                        })?;
3821                        if commands.is_empty()
3822                            && !workflow_context
3823                                .matched_recorded_pending()
3824                                .map_err(|error| {
3825                                    QueryTaskExecutionFailure::new(
3826                                        "query_workflow_state_unavailable",
3827                                        format!("workflow replay failed before query: {error}"),
3828                                        "QueryWorkflowStateUnavailable",
3829                                    )
3830                                })?
3831                        {
3832                            return Err(QueryTaskExecutionFailure::new(
3833                                "query_workflow_state_unavailable",
3834                                "workflow replay yielded without a durable command",
3835                                "QueryWorkflowStateUnavailable",
3836                            ));
3837                        }
3838                    }
3839                }
3840                let state = (invocation.snapshot)().map_err(|error| {
3841                    QueryTaskExecutionFailure::new(
3842                        "query_workflow_state_unavailable",
3843                        format!("cannot snapshot replayed workflow state: {error}"),
3844                        "QueryWorkflowStateUnavailable",
3845                    )
3846                })?;
3847                handler(context, state, args).map_err(|message| {
3848                    QueryTaskExecutionFailure::new(
3849                        "query_workflow_state_unavailable",
3850                        message,
3851                        "QueryWorkflowStateUnavailable",
3852                    )
3853                })?
3854            }
3855        };
3856
3857        future.await.map_err(|error| {
3858            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
3859        })
3860    }
3861
3862    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
3863        let workflow = self
3864            .workflows
3865            .get(&task.workflow_type)
3866            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
3867        let input = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3868        let resume_signal = decode_resume_signal(&task)?;
3869        let state = Arc::new(Mutex::new(WorkflowState::new_with_identity(
3870            task.history_events,
3871            task.workflow_id,
3872            task.run_id,
3873            self.task_queue.clone(),
3874            task.payload_codec.clone(),
3875            resume_signal,
3876        )?));
3877        let ctx = WorkflowContext { state };
3878        let mut future = (workflow.execute)(ctx.clone(), input);
3879        let mut cx = TaskContext::from_waker(noop_waker_ref());
3880
3881        match future.as_mut().poll(&mut cx) {
3882            Poll::Ready(Ok(result)) => {
3883                ctx.ensure_history_consumed()?;
3884                let result = encode_value_envelope(&result, &task.payload_codec)?;
3885                Ok(vec![json!({
3886                    "type": "complete_workflow",
3887                    "result": result
3888                })])
3889            }
3890            Poll::Ready(Err(error @ Error::ChildWorkflowFailed(_))) => {
3891                Ok(vec![workflow_failure_command(&error)])
3892            }
3893            Poll::Ready(Err(error)) => Err(error),
3894            Poll::Pending => {
3895                let commands = ctx.take_commands()?;
3896                if commands.is_empty() && !ctx.matched_recorded_pending()? {
3897                    Err(Error::WorkflowYieldedWithoutCommand)
3898                } else {
3899                    Ok(commands)
3900                }
3901            }
3902        }
3903    }
3904
3905    async fn execute_activity_task(&self, task: ActivityTask) -> Result<Value> {
3906        let handler = self
3907            .activities
3908            .get(&task.activity_type)
3909            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
3910        let args = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3911        let attempt_id = task
3912            .activity_attempt_id
3913            .clone()
3914            .or(task.attempt_id.clone())
3915            .unwrap_or_default();
3916        let lease_owner = task
3917            .lease_owner
3918            .clone()
3919            .unwrap_or_else(|| self.worker_id.clone());
3920        let ctx = ActivityContext {
3921            client: self.client.clone(),
3922            task_id: task.task_id,
3923            activity_attempt_id: attempt_id,
3924            lease_owner,
3925            activity_type: task.activity_type,
3926            attempt_number: task.attempt_number,
3927            task_queue: self.task_queue.clone(),
3928            worker_id: self.worker_id.clone(),
3929        };
3930
3931        handler(ctx, args).await
3932    }
3933}
3934
3935fn poller_result(
3936    kind: &str,
3937    result: std::result::Result<Result<()>, tokio::task::JoinError>,
3938) -> Result<()> {
3939    match result {
3940        Ok(result) => result,
3941        Err(error) => Err(Error::WorkerLoop(format!(
3942            "{kind} poller join error: {error}"
3943        ))),
3944    }
3945}
3946
3947fn optional_poller_result(
3948    kind: &str,
3949    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
3950) -> Result<()> {
3951    match result {
3952        Some(result) => poller_result(kind, result),
3953        None => Ok(()),
3954    }
3955}
3956
3957async fn join_pollers(
3958    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3959    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3960    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3961) -> Result<()> {
3962    let mut first_error = None;
3963
3964    if let Some(handle) = workflow_poller {
3965        if let Err(error) = poller_result("workflow", handle.await) {
3966            first_error.get_or_insert(error);
3967        }
3968    }
3969
3970    if let Some(handle) = activity_poller {
3971        if let Err(error) = poller_result("activity", handle.await) {
3972            first_error.get_or_insert(error);
3973        }
3974    }
3975
3976    if let Some(handle) = query_poller {
3977        if let Err(error) = poller_result("query", handle.await) {
3978            first_error.get_or_insert(error);
3979        }
3980    }
3981
3982    if let Some(error) = first_error {
3983        Err(error)
3984    } else {
3985        Ok(())
3986    }
3987}
3988
3989fn default_worker_id() -> String {
3990    let millis = SystemTime::now()
3991        .duration_since(UNIX_EPOCH)
3992        .unwrap_or_default()
3993        .as_millis();
3994    format!("rust-worker-{}-{millis}", std::process::id())
3995}
3996
3997fn unique_request_id(prefix: &str) -> String {
3998    let nanos = SystemTime::now()
3999        .duration_since(UNIX_EPOCH)
4000        .unwrap_or_default()
4001        .as_nanos();
4002    format!("{prefix}-{}-{nanos}", std::process::id())
4003}
4004
4005#[derive(Debug)]
4006struct QueryTaskExecutionFailure {
4007    reason: String,
4008    message: String,
4009    failure_type: String,
4010}
4011
4012impl QueryTaskExecutionFailure {
4013    fn new(
4014        reason: impl Into<String>,
4015        message: impl Into<String>,
4016        failure_type: impl Into<String>,
4017    ) -> Self {
4018        Self {
4019            reason: reason.into(),
4020            message: message.into(),
4021            failure_type: failure_type.into(),
4022        }
4023    }
4024}
4025
4026/// Typed local state owned by one deterministic workflow invocation.
4027///
4028/// Use [`WorkflowInstance::update`] for the same state transitions during
4029/// ordinary execution and replay. A replayed query receives a detached
4030/// immutable `Arc<S>` rather than this mutation-capable handle.
4031#[derive(Clone, Debug)]
4032pub struct WorkflowInstance<S> {
4033    state: Arc<Mutex<S>>,
4034}
4035
4036impl<S> WorkflowInstance<S> {
4037    fn new(state: S) -> Self {
4038        Self {
4039            state: Arc::new(Mutex::new(state)),
4040        }
4041    }
4042
4043    /// Read the current workflow-instance state without changing it.
4044    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
4045        let state = self
4046            .state
4047            .lock()
4048            .map_err(|_| Error::WorkflowStatePoisoned)?;
4049        Ok(reader(&state))
4050    }
4051
4052    /// Apply one deterministic workflow-instance state transition.
4053    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
4054        let mut state = self
4055            .state
4056            .lock()
4057            .map_err(|_| Error::WorkflowStatePoisoned)?;
4058        Ok(transition(&mut state))
4059    }
4060}
4061
4062impl<S: Clone> WorkflowInstance<S> {
4063    fn snapshot(&self) -> Result<S> {
4064        self.read(Clone::clone)
4065    }
4066}
4067
4068#[derive(Clone, Debug)]
4069pub struct WorkflowContext {
4070    state: Arc<Mutex<WorkflowState>>,
4071}
4072
4073impl WorkflowContext {
4074    /// Identity of the parent workflow currently being replayed.
4075    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
4076        let state = self
4077            .state
4078            .lock()
4079            .map_err(|_| Error::WorkflowStatePoisoned)?;
4080        Ok(WorkflowIdentity {
4081            workflow_id: state.workflow_id.clone(),
4082            run_id: state.run_id.clone(),
4083        })
4084    }
4085
4086    pub fn activity<T: Serialize>(
4087        &self,
4088        activity_type: impl Into<String>,
4089        args: T,
4090    ) -> ActivityCall {
4091        self.activity_with_options(activity_type, ActivityOptions::new(), args)
4092    }
4093
4094    pub fn activity_on_queue<T, Q>(
4095        &self,
4096        activity_type: impl Into<String>,
4097        task_queue: Option<Q>,
4098        args: T,
4099    ) -> ActivityCall
4100    where
4101        T: Serialize,
4102        Q: Into<String>,
4103    {
4104        let mut options = ActivityOptions::new();
4105        options.task_queue = task_queue.map(Into::into);
4106        self.activity_with_options(activity_type, options, args)
4107    }
4108
4109    /// Schedule one durable activity with retry, routing, and timeout options.
4110    ///
4111    /// Options are validated before the command is emitted. Once the command is
4112    /// recorded, replay consumes the same activity lifecycle at this command
4113    /// position and never emits a duplicate schedule.
4114    ///
4115    /// ```no_run
4116    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
4117    /// # use std::time::Duration;
4118    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
4119    /// let result = ctx
4120    ///     .activity_with_options(
4121    ///         "charge-card",
4122    ///         ActivityOptions::new()
4123    ///             .task_queue("payments")
4124    ///             .retry_policy(
4125    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
4126    ///                     Duration::from_secs(1),
4127    ///                     2,
4128    ///                     Some(Duration::from_secs(30)),
4129    ///                 ),
4130    ///             )
4131    ///             .start_to_close_timeout(Duration::from_secs(60))
4132    ///             .schedule_to_close_timeout(Duration::from_secs(180))
4133    ///             .heartbeat_timeout(Duration::from_secs(15)),
4134    ///         json!([{"order_id": "order-42"}]),
4135    ///     )
4136    ///     .await;
4137    /// match result {
4138    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
4139    ///         "reason": failure.reason,
4140    ///         "timeout_kind": failure.timeout_kind,
4141    ///     })),
4142    ///     other => other,
4143    /// }
4144    /// # }
4145    /// ```
4146    pub fn activity_with_options<T: Serialize>(
4147        &self,
4148        activity_type: impl Into<String>,
4149        options: ActivityOptions,
4150        args: T,
4151    ) -> ActivityCall {
4152        ActivityCall {
4153            ctx: self.clone(),
4154            activity_type: activity_type.into(),
4155            options,
4156            args: Some(serde_json::to_value(args).map_err(Error::from)),
4157            scheduled: false,
4158        }
4159    }
4160
4161    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
4162        SignalCall {
4163            ctx: self.clone(),
4164            signal_name: signal_name.into(),
4165            opened_wait: false,
4166            matched_pending: false,
4167        }
4168    }
4169
4170    /// Wait for server-backed durable time without blocking the worker executor.
4171    ///
4172    /// Polling this future emits one `start_timer` command and yields. The
4173    /// server records the deadline, so neither worker nor server restarts reset
4174    /// the wait. Replay resolves the future only from a `TimerScheduled` and
4175    /// `TimerFired` pair at the same position in the shared durable-command
4176    /// stream, with matching sequence, timer identity, and delay. Sub-second
4177    /// durations round up because protocol deadlines use whole seconds.
4178    ///
4179    /// ```no_run
4180    /// # use durable_workflow::{json, Client, Worker};
4181    /// # use std::time::Duration;
4182    /// # fn configure(client: Client) {
4183    /// let mut worker = Worker::new(client, "rust-workers");
4184    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
4185    ///     ctx.sleep(Duration::from_secs(5)).await?;
4186    ///     Ok(json!({"status": "timer fired"}))
4187    /// });
4188    /// # }
4189    /// ```
4190    pub fn sleep(&self, duration: Duration) -> TimerCall {
4191        let delay_seconds = duration
4192            .as_secs()
4193            .checked_add(u64::from(duration.subsec_nanos() > 0));
4194        TimerCall {
4195            ctx: self.clone(),
4196            delay_seconds,
4197            scheduled: false,
4198            matched_pending: false,
4199        }
4200    }
4201
4202    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
4203    pub fn start_timer(&self, duration: Duration) -> TimerCall {
4204        self.sleep(duration)
4205    }
4206
4207    /// Start a named durable child on an explicit queue and await its result.
4208    ///
4209    /// The command is recorded in the parent's sequence-ordered durable command
4210    /// stream. Replay keeps a scheduled child pending without emitting another
4211    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
4212    /// values preserve the history payload codec and include both sides of the
4213    /// durable relationship; failures are returned as
4214    /// [`Error::ChildWorkflowFailed`].
4215    ///
4216    /// ```no_run
4217    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
4218    /// # fn configure(client: Client) {
4219    /// let mut worker = Worker::new(client, "parent-workers");
4220    /// worker.register_workflow("order-parent", |ctx, _input| async move {
4221    ///     let child = ctx
4222    ///         .start_child_workflow(
4223    ///             "fulfil-order",
4224    ///             ChildWorkflowOptions::new("fulfilment-workers")
4225    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
4226    ///             json!([{"order_id": "order-42"}]),
4227    ///         )
4228    ///         .await?;
4229    ///     Ok(child.result)
4230    /// });
4231    /// # }
4232    /// ```
4233    pub fn start_child_workflow<T: Serialize>(
4234        &self,
4235        workflow_type: impl Into<String>,
4236        options: ChildWorkflowOptions,
4237        args: T,
4238    ) -> ChildWorkflowCall {
4239        ChildWorkflowCall {
4240            ctx: self.clone(),
4241            workflow_type: workflow_type.into(),
4242            options,
4243            args: Some(serde_json::to_value(args).map_err(Error::from)),
4244            scheduled: false,
4245            matched_pending: false,
4246        }
4247    }
4248
4249    fn take_commands(&self) -> Result<Vec<Value>> {
4250        let mut state = self
4251            .state
4252            .lock()
4253            .map_err(|_| Error::WorkflowStatePoisoned)?;
4254        Ok(std::mem::take(&mut state.commands))
4255    }
4256
4257    fn matched_recorded_pending(&self) -> Result<bool> {
4258        let state = self
4259            .state
4260            .lock()
4261            .map_err(|_| Error::WorkflowStatePoisoned)?;
4262        Ok(state.matched_recorded_pending)
4263    }
4264
4265    fn ensure_history_consumed(&self) -> Result<()> {
4266        let state = self
4267            .state
4268            .lock()
4269            .map_err(|_| Error::WorkflowStatePoisoned)?;
4270        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
4271            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
4272                "recorded_commands_unconsumed",
4273                Some(command.sequence()),
4274                Some(command.shape().to_string()),
4275                Some("workflow completion".to_string()),
4276                "workflow completed before consuming all recorded durable commands",
4277            )));
4278        }
4279        Ok(())
4280    }
4281}
4282
4283#[derive(Debug)]
4284struct WorkflowState {
4285    history: Vec<HistoryEvent>,
4286    workflow_id: Option<String>,
4287    run_id: Option<String>,
4288    task_queue: String,
4289    payload_codec: String,
4290    resume_signal: Option<ResumeSignal>,
4291    recorded_commands: Vec<RecordedCommand>,
4292    command_cursor: usize,
4293    matched_recorded_pending: bool,
4294    signal_cursors: HashMap<String, usize>,
4295    commands: Vec<Value>,
4296}
4297
4298impl WorkflowState {
4299    #[cfg(test)]
4300    fn new(
4301        history: Vec<HistoryEvent>,
4302        task_queue: String,
4303        payload_codec: String,
4304        resume_signal: Option<ResumeSignal>,
4305    ) -> Result<Self> {
4306        Self::new_with_identity(
4307            history,
4308            None,
4309            None,
4310            task_queue,
4311            payload_codec,
4312            resume_signal,
4313        )
4314    }
4315
4316    fn new_with_identity(
4317        history: Vec<HistoryEvent>,
4318        workflow_id: Option<String>,
4319        run_id: Option<String>,
4320        task_queue: String,
4321        payload_codec: String,
4322        resume_signal: Option<ResumeSignal>,
4323    ) -> Result<Self> {
4324        let recorded_commands = recorded_commands(
4325            &history,
4326            &payload_codec,
4327            WorkflowIdentity {
4328                workflow_id: workflow_id.clone(),
4329                run_id: run_id.clone(),
4330            },
4331        )?;
4332        Ok(Self {
4333            history,
4334            workflow_id,
4335            run_id,
4336            task_queue,
4337            payload_codec,
4338            resume_signal,
4339            recorded_commands,
4340            command_cursor: 0,
4341            matched_recorded_pending: false,
4342            signal_cursors: HashMap::new(),
4343            commands: Vec::new(),
4344        })
4345    }
4346}
4347
4348#[derive(Clone, Debug)]
4349enum RecordedCommand {
4350    Activity {
4351        sequence: u64,
4352        activity_type: Option<String>,
4353        options: Option<RecordedActivityOptions>,
4354        outcome: Option<ActivityOutcome>,
4355    },
4356    Timer {
4357        sequence: u64,
4358        delay_seconds: u64,
4359        fired: bool,
4360    },
4361    ChildWorkflow {
4362        sequence: u64,
4363        workflow_type: Option<String>,
4364        outcome: Option<ChildWorkflowOutcome>,
4365    },
4366    SignalWait {
4367        sequence: u64,
4368        signal_name: Option<String>,
4369    },
4370}
4371
4372#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4373struct RecordedActivityOptions {
4374    task_queue: RecordedSnapshotValue<Option<String>>,
4375    execution_mode: RecordedSnapshotValue<Option<String>>,
4376    retry_policy: ActivityRetrySnapshot,
4377}
4378
4379#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4380enum RecordedSnapshotValue<T> {
4381    /// Older history did not persist this field, so it cannot constrain replay.
4382    Unknown,
4383    Known(T),
4384}
4385
4386impl<T: PartialEq> RecordedSnapshotValue<T> {
4387    fn matches_current(&self, current: &Self) -> bool {
4388        match self {
4389            Self::Unknown => true,
4390            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
4391        }
4392    }
4393}
4394
4395#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4396struct ActivityRetrySnapshot {
4397    snapshot_version: RecordedSnapshotValue<Option<u64>>,
4398    max_attempts: RecordedSnapshotValue<Option<u64>>,
4399    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
4400    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4401    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
4402    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4403    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
4404    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
4405}
4406
4407impl ActivityRetrySnapshot {
4408    fn matches_current(&self, current: &Self) -> bool {
4409        self.snapshot_version
4410            .matches_current(&current.snapshot_version)
4411            && self.max_attempts.matches_current(&current.max_attempts)
4412            && self
4413                .backoff_seconds
4414                .matches_current(&current.backoff_seconds)
4415            && self
4416                .start_to_close_timeout
4417                .matches_current(&current.start_to_close_timeout)
4418            && self
4419                .schedule_to_start_timeout
4420                .matches_current(&current.schedule_to_start_timeout)
4421            && self
4422                .schedule_to_close_timeout
4423                .matches_current(&current.schedule_to_close_timeout)
4424            && self
4425                .heartbeat_timeout
4426                .matches_current(&current.heartbeat_timeout)
4427            && self
4428                .non_retryable_error_types
4429                .matches_current(&current.non_retryable_error_types)
4430    }
4431}
4432
4433fn recorded_optional_u64(
4434    object: Option<&serde_json::Map<String, Value>>,
4435    field: &str,
4436) -> RecordedSnapshotValue<Option<u64>> {
4437    match object.and_then(|object| object.get(field)) {
4438        None => RecordedSnapshotValue::Unknown,
4439        Some(Value::Null) => RecordedSnapshotValue::Known(None),
4440        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
4441    }
4442}
4443
4444fn recorded_optional_string(
4445    object: &serde_json::Map<String, Value>,
4446    field: &str,
4447) -> RecordedSnapshotValue<Option<String>> {
4448    match object.get(field) {
4449        None => RecordedSnapshotValue::Unknown,
4450        Some(Value::Null) => RecordedSnapshotValue::Known(None),
4451        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
4452    }
4453}
4454
4455fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
4456    let policy = policy.and_then(Value::as_object);
4457    let backoff_seconds = policy
4458        .and_then(|policy| policy.get("backoff_seconds"))
4459        .and_then(Value::as_array)
4460        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4461        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
4462    let mut non_retryable_error_types = Vec::new();
4463    for error_type in policy
4464        .and_then(|policy| policy.get("non_retryable_error_types"))
4465        .and_then(Value::as_array)
4466        .into_iter()
4467        .flatten()
4468        .filter_map(Value::as_str)
4469        .map(str::trim)
4470        .filter(|error_type| !error_type.is_empty())
4471    {
4472        if !non_retryable_error_types
4473            .iter()
4474            .any(|recorded| recorded == error_type)
4475        {
4476            non_retryable_error_types.push(error_type.to_string());
4477        }
4478    }
4479
4480    ActivityRetrySnapshot {
4481        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
4482        max_attempts: recorded_optional_u64(policy, "max_attempts"),
4483        backoff_seconds,
4484        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
4485        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
4486        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
4487        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
4488        non_retryable_error_types: if policy
4489            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
4490        {
4491            RecordedSnapshotValue::Known(non_retryable_error_types)
4492        } else {
4493            RecordedSnapshotValue::Unknown
4494        },
4495    }
4496}
4497
4498fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
4499    let policy = options.retry_policy.as_ref();
4500    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
4501        Some(Value::Null) => None,
4502        Some(value) => value_as_u64(value),
4503        None => Some(1),
4504    };
4505    let backoff_seconds = policy
4506        .and_then(|policy| policy.get("backoff_seconds"))
4507        .and_then(Value::as_array)
4508        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4509        .unwrap_or_default();
4510    let non_retryable_error_types = policy
4511        .and_then(|policy| policy.get("non_retryable_error_types"))
4512        .and_then(Value::as_array)
4513        .into_iter()
4514        .flatten()
4515        .filter_map(Value::as_str)
4516        .map(str::to_string)
4517        .collect();
4518
4519    ActivityRetrySnapshot {
4520        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
4521        max_attempts: RecordedSnapshotValue::Known(max_attempts),
4522        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
4523        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
4524        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
4525        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
4526        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
4527        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
4528    }
4529}
4530
4531fn activity_options_description(options: &RecordedActivityOptions) -> String {
4532    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
4533}
4534
4535impl RecordedCommand {
4536    fn sequence(&self) -> u64 {
4537        match self {
4538            Self::Activity { sequence, .. }
4539            | Self::Timer { sequence, .. }
4540            | Self::ChildWorkflow { sequence, .. }
4541            | Self::SignalWait { sequence, .. } => *sequence,
4542        }
4543    }
4544
4545    fn shape(&self) -> &'static str {
4546        match self {
4547            Self::Activity { .. } => "activity",
4548            Self::Timer { .. } => "timer",
4549            Self::ChildWorkflow { .. } => "child workflow",
4550            Self::SignalWait { .. } => "signal wait",
4551        }
4552    }
4553}
4554
4555#[derive(Clone, Debug)]
4556struct ResumeSignal {
4557    signal_id: Option<String>,
4558    signal_name: String,
4559    arguments: Vec<Value>,
4560}
4561
4562pub struct ActivityCall {
4563    ctx: WorkflowContext,
4564    activity_type: String,
4565    options: ActivityOptions,
4566    args: Option<Result<Value>>,
4567    scheduled: bool,
4568}
4569
4570impl Future for ActivityCall {
4571    type Output = Result<Value>;
4572
4573    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4574        let ctx = self.ctx.clone();
4575        let mut state = match ctx.state.lock() {
4576            Ok(state) => state,
4577            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4578        };
4579
4580        if self.scheduled {
4581            return Poll::Pending;
4582        }
4583
4584        let options = match self.options.validate() {
4585            Ok(options) => options,
4586            Err(error) => {
4587                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
4588            }
4589        };
4590        let task_queue = options
4591            .task_queue
4592            .clone()
4593            .unwrap_or_else(|| state.task_queue.clone());
4594        let current_recorded_options = RecordedActivityOptions {
4595            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
4596            // Rust schedules ordinary durable activities. The server records a
4597            // non-null mode only for a specialized execution primitive.
4598            execution_mode: RecordedSnapshotValue::Known(None),
4599            retry_policy: current_activity_retry_snapshot(&options),
4600        };
4601
4602        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4603            let sequence = recorded.sequence();
4604            match recorded {
4605                RecordedCommand::Activity {
4606                    activity_type,
4607                    options: recorded_options,
4608                    outcome,
4609                    ..
4610                } => {
4611                    if let Some(recorded_type) = activity_type {
4612                        if recorded_type != self.activity_type {
4613                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4614                                ReplayFailure::new(
4615                                    "recorded_command_detail_mismatch",
4616                                    Some(sequence),
4617                                    Some(format!("activity:{recorded_type}")),
4618                                    Some(format!("activity:{}", self.activity_type)),
4619                                    "recorded activity type differs from the current workflow command",
4620                                ),
4621                            )));
4622                        }
4623                    }
4624                    if let Some(recorded_options) = recorded_options {
4625                        if !recorded_options
4626                            .task_queue
4627                            .matches_current(&current_recorded_options.task_queue)
4628                        {
4629                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4630                                ReplayFailure::new(
4631                                    "activity_task_queue_mismatch",
4632                                    Some(sequence),
4633                                    Some(activity_options_description(&recorded_options)),
4634                                    Some(activity_options_description(&current_recorded_options)),
4635                                    "recorded activity task queue differs from the current workflow command",
4636                                ),
4637                            )));
4638                        }
4639                        if !recorded_options
4640                            .execution_mode
4641                            .matches_current(&current_recorded_options.execution_mode)
4642                        {
4643                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4644                                ReplayFailure::new(
4645                                    "activity_execution_mode_mismatch",
4646                                    Some(sequence),
4647                                    Some(activity_options_description(&recorded_options)),
4648                                    Some(activity_options_description(&current_recorded_options)),
4649                                    "recorded activity execution mode differs from the current workflow command",
4650                                ),
4651                            )));
4652                        }
4653                        if !recorded_options
4654                            .retry_policy
4655                            .matches_current(&current_recorded_options.retry_policy)
4656                        {
4657                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4658                                ReplayFailure::new(
4659                                    "activity_retry_policy_mismatch",
4660                                    Some(sequence),
4661                                    Some(activity_options_description(&recorded_options)),
4662                                    Some(activity_options_description(&current_recorded_options)),
4663                                    "recorded activity retry policy differs from the current workflow command",
4664                                ),
4665                            )));
4666                        }
4667                    }
4668                    state.command_cursor += 1;
4669                    if let Some(outcome) = outcome {
4670                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
4671                    }
4672                    state.matched_recorded_pending = true;
4673                    self.scheduled = true;
4674                    return Poll::Pending;
4675                }
4676                other => {
4677                    return Poll::Ready(Err(command_mismatch(
4678                        &other,
4679                        format!("activity:{}", self.activity_type),
4680                    )));
4681                }
4682            }
4683        }
4684
4685        if !self.scheduled {
4686            let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4687                Ok(args) => args,
4688                Err(error) => return Poll::Ready(Err(error)),
4689            };
4690            let arguments = normalize_arguments(args);
4691            let envelope = match encode_value_envelope(&arguments, &state.payload_codec) {
4692                Ok(envelope) => envelope,
4693                Err(error) => return Poll::Ready(Err(error)),
4694            };
4695
4696            let mut command = serde_json::Map::from_iter([
4697                ("type".to_string(), json!("schedule_activity")),
4698                (
4699                    "activity_type".to_string(),
4700                    json!(self.activity_type.clone()),
4701                ),
4702                ("queue".to_string(), json!(task_queue)),
4703                ("arguments".to_string(), envelope),
4704            ]);
4705            for (field, value) in [
4706                ("start_to_close_timeout", options.start_to_close_timeout),
4707                (
4708                    "schedule_to_start_timeout",
4709                    options.schedule_to_start_timeout,
4710                ),
4711                (
4712                    "schedule_to_close_timeout",
4713                    options.schedule_to_close_timeout,
4714                ),
4715                ("heartbeat_timeout", options.heartbeat_timeout),
4716            ] {
4717                if let Some(value) = value {
4718                    command.insert(field.to_string(), json!(value));
4719                }
4720            }
4721            if let Some(retry_policy) = options.retry_policy {
4722                command.insert("retry_policy".to_string(), retry_policy);
4723            }
4724            state.commands.push(Value::Object(command));
4725            self.scheduled = true;
4726        }
4727
4728        Poll::Pending
4729    }
4730}
4731
4732/// Future returned by [`WorkflowContext::sleep`].
4733pub struct TimerCall {
4734    ctx: WorkflowContext,
4735    delay_seconds: Option<u64>,
4736    scheduled: bool,
4737    matched_pending: bool,
4738}
4739
4740impl Future for TimerCall {
4741    type Output = Result<()>;
4742
4743    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4744        if self.matched_pending {
4745            return Poll::Pending;
4746        }
4747
4748        let ctx = self.ctx.clone();
4749        let Some(requested_delay) = self.delay_seconds else {
4750            return Poll::Ready(Err(Error::TimerDurationOverflow));
4751        };
4752        let mut state = match ctx.state.lock() {
4753            Ok(state) => state,
4754            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4755        };
4756
4757        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4758            match recorded {
4759                RecordedCommand::Timer {
4760                    sequence,
4761                    delay_seconds,
4762                    fired,
4763                    ..
4764                } => {
4765                    if delay_seconds != requested_delay {
4766                        return Poll::Ready(Err(Error::NonDeterministicReplay(
4767                            ReplayFailure::new(
4768                                "timer_delay_mismatch",
4769                                Some(sequence),
4770                                Some(format!("timer:{delay_seconds}s")),
4771                                Some(format!("timer:{requested_delay}s")),
4772                                "recorded timer delay differs from the current workflow command",
4773                            ),
4774                        )));
4775                    }
4776                    state.command_cursor += 1;
4777                    if fired {
4778                        return Poll::Ready(Ok(()));
4779                    }
4780                    state.matched_recorded_pending = true;
4781                    self.scheduled = true;
4782                    self.matched_pending = true;
4783                    return Poll::Pending;
4784                }
4785                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
4786            }
4787        }
4788
4789        if !self.scheduled {
4790            state.commands.push(json!({
4791                "type": "start_timer",
4792                "delay_seconds": requested_delay,
4793            }));
4794            self.scheduled = true;
4795        }
4796
4797        Poll::Pending
4798    }
4799}
4800
4801/// Future returned by [`WorkflowContext::start_child_workflow`].
4802pub struct ChildWorkflowCall {
4803    ctx: WorkflowContext,
4804    workflow_type: String,
4805    options: ChildWorkflowOptions,
4806    args: Option<Result<Value>>,
4807    scheduled: bool,
4808    matched_pending: bool,
4809}
4810
4811impl Future for ChildWorkflowCall {
4812    type Output = Result<ChildWorkflowResult>;
4813
4814    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4815        if self.matched_pending {
4816            return Poll::Pending;
4817        }
4818
4819        let ctx = self.ctx.clone();
4820        let mut state = match ctx.state.lock() {
4821            Ok(state) => state,
4822            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4823        };
4824
4825        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4826            let sequence = recorded.sequence();
4827            match recorded {
4828                RecordedCommand::ChildWorkflow {
4829                    workflow_type,
4830                    outcome,
4831                    ..
4832                } => {
4833                    if let Some(recorded_type) = workflow_type {
4834                        if recorded_type != self.workflow_type {
4835                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4836                                ReplayFailure::new(
4837                                    "recorded_command_detail_mismatch",
4838                                    Some(sequence),
4839                                    Some(format!("child workflow:{recorded_type}")),
4840                                    Some(format!("child workflow:{}", self.workflow_type)),
4841                                    "recorded child workflow type differs from the current workflow command",
4842                                ),
4843                            )));
4844                        }
4845                    }
4846                    state.command_cursor += 1;
4847                    if let Some(outcome) = outcome {
4848                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
4849                    }
4850                    state.matched_recorded_pending = true;
4851                    self.scheduled = true;
4852                    self.matched_pending = true;
4853                    return Poll::Pending;
4854                }
4855                other => {
4856                    return Poll::Ready(Err(command_mismatch(
4857                        &other,
4858                        format!("child workflow:{}", self.workflow_type),
4859                    )));
4860                }
4861            }
4862        }
4863
4864        if !self.scheduled {
4865            if self.options.task_queue.trim().is_empty() {
4866                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4867                    "task_queue must not be empty".to_string(),
4868                )));
4869            }
4870            for (name, value) in [
4871                (
4872                    "execution_timeout_seconds",
4873                    self.options.execution_timeout_seconds,
4874                ),
4875                ("run_timeout_seconds", self.options.run_timeout_seconds),
4876            ] {
4877                if value == Some(0) {
4878                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
4879                        "{name} must be at least 1"
4880                    ))));
4881                }
4882            }
4883
4884            let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4885                Ok(args) => args,
4886                Err(error) => return Poll::Ready(Err(error)),
4887            };
4888            let arguments =
4889                match encode_value_envelope(&normalize_arguments(args), &state.payload_codec) {
4890                    Ok(arguments) => arguments,
4891                    Err(error) => return Poll::Ready(Err(error)),
4892                };
4893            let mut command = json!({
4894                "type": "start_child_workflow",
4895                "workflow_type": self.workflow_type,
4896                "queue": self.options.task_queue,
4897                "parent_close_policy": self.options.parent_close_policy.as_str(),
4898                "arguments": arguments,
4899            });
4900            let object = command
4901                .as_object_mut()
4902                .expect("child workflow command is always an object");
4903            if let Some(policy) = &self.options.retry_policy {
4904                let mut retry_policy = serde_json::Map::new();
4905                if let Some(max_attempts) = policy.max_attempts {
4906                    if max_attempts == 0 {
4907                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4908                            "retry_policy.max_attempts must be at least 1".to_string(),
4909                        )));
4910                    }
4911                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
4912                }
4913                if !policy.backoff_seconds.is_empty() {
4914                    retry_policy
4915                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
4916                }
4917                if !policy.non_retryable_error_types.is_empty() {
4918                    retry_policy.insert(
4919                        "non_retryable_error_types".to_string(),
4920                        json!(policy.non_retryable_error_types),
4921                    );
4922                }
4923                if retry_policy.is_empty() {
4924                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4925                        "retry_policy must configure at least one field".to_string(),
4926                    )));
4927                }
4928                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
4929            }
4930            if let Some(seconds) = self.options.execution_timeout_seconds {
4931                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
4932            }
4933            if let Some(seconds) = self.options.run_timeout_seconds {
4934                object.insert("run_timeout_seconds".to_string(), json!(seconds));
4935            }
4936            state.commands.push(command);
4937            self.scheduled = true;
4938        }
4939
4940        Poll::Pending
4941    }
4942}
4943
4944fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
4945    Error::NonDeterministicReplay(ReplayFailure::new(
4946        "recorded_command_mismatch",
4947        Some(recorded.sequence()),
4948        Some(recorded.shape().to_string()),
4949        Some(actual.into()),
4950        "current workflow command does not match the recorded durable command sequence",
4951    ))
4952}
4953
4954pub struct SignalCall {
4955    ctx: WorkflowContext,
4956    signal_name: String,
4957    opened_wait: bool,
4958    matched_pending: bool,
4959}
4960
4961impl Future for SignalCall {
4962    type Output = Result<Vec<Value>>;
4963
4964    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4965        if self.matched_pending {
4966            return Poll::Pending;
4967        }
4968
4969        let ctx = self.ctx.clone();
4970        let mut state = match ctx.state.lock() {
4971            Ok(state) => state,
4972            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4973        };
4974
4975        let signals = match signal_values(
4976            &state.history,
4977            &self.signal_name,
4978            &state.payload_codec,
4979            state.resume_signal.as_ref(),
4980        ) {
4981            Ok(signals) => signals,
4982            Err(error) => return Poll::Ready(Err(error)),
4983        };
4984        let cursor = *state.signal_cursors.get(&self.signal_name).unwrap_or(&0);
4985
4986        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4987            match recorded {
4988                RecordedCommand::SignalWait {
4989                    sequence,
4990                    signal_name,
4991                } => {
4992                    if let Some(recorded_name) = signal_name {
4993                        if recorded_name != self.signal_name {
4994                            return Poll::Ready(Err(Error::NonDeterministicReplay(
4995                                ReplayFailure::new(
4996                                    "recorded_command_detail_mismatch",
4997                                    Some(sequence),
4998                                    Some(format!("signal wait:{recorded_name}")),
4999                                    Some(format!("signal wait:{}", self.signal_name)),
5000                                    "recorded signal name differs from the current workflow command",
5001                                ),
5002                            )));
5003                        }
5004                    }
5005
5006                    state.command_cursor += 1;
5007                    if cursor < signals.len() {
5008                        state
5009                            .signal_cursors
5010                            .insert(self.signal_name.clone(), cursor + 1);
5011                        return Poll::Ready(Ok(signals[cursor].clone()));
5012                    }
5013
5014                    state.matched_recorded_pending = true;
5015                    self.opened_wait = true;
5016                    self.matched_pending = true;
5017                    return Poll::Pending;
5018                }
5019                other => {
5020                    return Poll::Ready(Err(command_mismatch(
5021                        &other,
5022                        format!("signal wait:{}", self.signal_name),
5023                    )));
5024                }
5025            }
5026        }
5027
5028        if cursor < signals.len() {
5029            state
5030                .signal_cursors
5031                .insert(self.signal_name.clone(), cursor + 1);
5032            return Poll::Ready(Ok(signals[cursor].clone()));
5033        }
5034
5035        if !self.opened_wait {
5036            state.commands.push(json!({
5037                "type": "open_condition_wait",
5038                "condition_key": format!("signal:{}", self.signal_name)
5039            }));
5040            self.opened_wait = true;
5041        }
5042
5043        Poll::Pending
5044    }
5045}
5046
5047#[derive(Clone, Debug)]
5048pub struct ActivityContext {
5049    client: Client,
5050    pub task_id: String,
5051    pub activity_attempt_id: String,
5052    pub lease_owner: String,
5053    pub activity_type: String,
5054    pub attempt_number: u64,
5055    pub task_queue: String,
5056    pub worker_id: String,
5057}
5058
5059impl ActivityContext {
5060    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
5061        self.client
5062            .heartbeat_activity_task(
5063                &self.task_id,
5064                &self.activity_attempt_id,
5065                &self.lease_owner,
5066                serde_json::to_value(details)?,
5067            )
5068            .await
5069    }
5070}
5071
5072fn decode_task_arguments(value: Option<&Value>, codec: &str) -> Result<Value> {
5073    match value {
5074        Some(value) => Ok(normalize_arguments(decode_wire_value(value, codec)?)),
5075        None => Ok(Value::Array(Vec::new())),
5076    }
5077}
5078
5079fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
5080    let Some(signal_name) = task
5081        .signal_name
5082        .as_deref()
5083        .filter(|value| !value.is_empty())
5084    else {
5085        return Ok(None);
5086    };
5087    let Some(arguments) = task.signal_arguments.as_ref() else {
5088        return Ok(None);
5089    };
5090
5091    let decoded = normalize_arguments(decode_wire_value(arguments, &task.payload_codec)?);
5092    let Value::Array(arguments) = decoded else {
5093        unreachable!("normalize_arguments always returns an array");
5094    };
5095
5096    Ok(Some(ResumeSignal {
5097        signal_id: task.workflow_signal_id.clone(),
5098        signal_name: signal_name.to_string(),
5099        arguments,
5100    }))
5101}
5102
5103fn normalize_arguments(value: Value) -> Value {
5104    match value {
5105        Value::Null => Value::Array(Vec::new()),
5106        Value::Array(_) => value,
5107        other => Value::Array(vec![other]),
5108    }
5109}
5110
5111fn recorded_commands(
5112    events: &[HistoryEvent],
5113    fallback_codec: &str,
5114    parent: WorkflowIdentity,
5115) -> Result<Vec<RecordedCommand>> {
5116    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
5117
5118    for event in events {
5119        let is_activity = matches!(
5120            event.event_type.as_str(),
5121            "ActivityScheduled"
5122                | "ActivityStarted"
5123                | "ActivityHeartbeatRecorded"
5124                | "ActivityRetryScheduled"
5125                | "ActivityCompleted"
5126                | "ActivityFailed"
5127                | "ActivityCancelled"
5128                | "ActivityTimedOut"
5129        );
5130        let is_workflow_timer = matches!(
5131            event.event_type.as_str(),
5132            "TimerScheduled" | "TimerCancelled" | "TimerFired"
5133        ) && !is_internal_timer_event(event);
5134        let is_child_workflow = matches!(
5135            event.event_type.as_str(),
5136            "ChildWorkflowScheduled"
5137                | "ChildRunCompleted"
5138                | "ChildRunFailed"
5139                | "ChildRunCancelled"
5140                | "ChildRunTerminated"
5141        );
5142        let is_signal_wait = is_recorded_signal_wait_event(event);
5143        if !is_activity && !is_workflow_timer && !is_child_workflow && !is_signal_wait {
5144            continue;
5145        }
5146
5147        let sequence = durable_event_sequence(event).ok_or_else(|| {
5148            Error::NonDeterministicReplay(ReplayFailure::new(
5149                "durable_command_sequence_missing",
5150                None,
5151                Some("positive workflow sequence".to_string()),
5152                Some(event.event_type.clone()),
5153                "durable command history event has no workflow sequence",
5154            ))
5155        })?;
5156        if sequence == 0 {
5157            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5158                "durable_command_sequence_invalid",
5159                Some(sequence),
5160                Some("positive workflow sequence".to_string()),
5161                Some(sequence.to_string()),
5162                "durable command history uses an invalid workflow sequence",
5163            )));
5164        }
5165        events_by_sequence.entry(sequence).or_default().push(event);
5166    }
5167
5168    events_by_sequence
5169        .into_iter()
5170        .map(|(sequence, sequence_events)| {
5171            let activity_events: Vec<_> = sequence_events
5172                .iter()
5173                .copied()
5174                .filter(|event| event.event_type.starts_with("Activity"))
5175                .collect();
5176            let timer_events: Vec<_> = sequence_events
5177                .iter()
5178                .copied()
5179                .filter(|event| event.event_type.starts_with("Timer"))
5180                .collect();
5181            let child_events: Vec<_> = sequence_events
5182                .iter()
5183                .copied()
5184                .filter(|event| {
5185                    event.event_type == "ChildWorkflowScheduled"
5186                        || event.event_type.starts_with("ChildRun")
5187                })
5188                .collect();
5189            let signal_wait_events: Vec<_> = sequence_events
5190                .iter()
5191                .copied()
5192                .filter(|event| is_recorded_signal_wait_event(event))
5193                .collect();
5194
5195            let command_kind_count = usize::from(!activity_events.is_empty())
5196                + usize::from(!timer_events.is_empty())
5197                + usize::from(!child_events.is_empty())
5198                + usize::from(!signal_wait_events.is_empty());
5199            if command_kind_count > 1 {
5200                let actual = [
5201                    (!activity_events.is_empty()).then_some("activity"),
5202                    (!timer_events.is_empty()).then_some("timer"),
5203                    (!child_events.is_empty()).then_some("child workflow"),
5204                    (!signal_wait_events.is_empty()).then_some("signal wait"),
5205                ]
5206                .into_iter()
5207                .flatten()
5208                .collect::<Vec<_>>()
5209                .join(" and ");
5210                return Err(invalid_recorded_history(
5211                    "durable_command_sequence_collision",
5212                    sequence,
5213                    "one durable command kind",
5214                    &actual,
5215                    "one workflow sequence records more than one durable command kind",
5216                ));
5217            }
5218
5219            if !activity_events.is_empty() {
5220                let scheduled_count = activity_events
5221                    .iter()
5222                    .filter(|event| event.event_type == "ActivityScheduled")
5223                    .count();
5224                if scheduled_count > 1 {
5225                    return Err(invalid_recorded_history(
5226                        "duplicate_activity_schedule",
5227                        sequence,
5228                        "at most one ActivityScheduled event",
5229                        "multiple ActivityScheduled events",
5230                        "activity history schedules more than one command at one workflow sequence",
5231                    ));
5232                }
5233                let activity_type = activity_events.iter().find_map(|event| {
5234                    event
5235                        .payload
5236                        .get("activity_type")
5237                        .or_else(|| event.payload.get("activity_name"))
5238                        .and_then(Value::as_str)
5239                        .map(str::to_string)
5240                });
5241                if activity_events.iter().filter_map(|event| {
5242                    event
5243                        .payload
5244                        .get("activity_type")
5245                        .or_else(|| event.payload.get("activity_name"))
5246                        .and_then(Value::as_str)
5247                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
5248                    return Err(invalid_recorded_history(
5249                        "activity_identity_mismatch",
5250                        sequence,
5251                        activity_type.as_deref().unwrap_or("one activity identity"),
5252                        "conflicting activity identities",
5253                        "activity lifecycle events at one workflow sequence disagree on identity",
5254                    ));
5255                }
5256                let terminal: Vec<_> = activity_events
5257                    .iter()
5258                    .copied()
5259                    .filter(|event| {
5260                        matches!(
5261                            event.event_type.as_str(),
5262                            "ActivityCompleted"
5263                                | "ActivityFailed"
5264                                | "ActivityCancelled"
5265                                | "ActivityTimedOut"
5266                        )
5267                    })
5268                    .collect();
5269                if terminal.len() > 1 {
5270                    return Err(invalid_recorded_history(
5271                        "duplicate_activity_terminal_event",
5272                        sequence,
5273                        "at most one terminal activity event",
5274                        "multiple terminal activity events",
5275                        "activity history settles one command more than once",
5276                    ));
5277                }
5278                let outcome = terminal
5279                    .first()
5280                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
5281                    .transpose()?;
5282                let options = activity_events
5283                    .iter()
5284                    .find(|event| event.event_type == "ActivityScheduled")
5285                    .and_then(|event| event.payload.get("activity"))
5286                    .and_then(Value::as_object)
5287                    .map(|activity| RecordedActivityOptions {
5288                        task_queue: recorded_optional_string(activity, "queue"),
5289                        execution_mode: recorded_optional_string(activity, "execution_mode"),
5290                        retry_policy: recorded_activity_retry_snapshot(
5291                            activity.get("retry_policy"),
5292                        ),
5293                    });
5294                return Ok(RecordedCommand::Activity {
5295                    sequence,
5296                    activity_type,
5297                    options,
5298                    outcome,
5299                });
5300            }
5301
5302            if !child_events.is_empty() {
5303                let scheduled: Vec<_> = child_events
5304                    .iter()
5305                    .copied()
5306                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
5307                    .collect();
5308                if scheduled.len() != 1 {
5309                    return Err(invalid_recorded_history(
5310                        "child_workflow_schedule_missing_or_duplicate",
5311                        sequence,
5312                        "one ChildWorkflowScheduled event",
5313                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
5314                        "child workflow replay requires exactly one recorded schedule event",
5315                    ));
5316                }
5317                let workflow_type = child_events.iter().find_map(|event| {
5318                    event
5319                        .payload
5320                        .get("child_workflow_type")
5321                        .or_else(|| event.payload.get("workflow_type"))
5322                        .and_then(Value::as_str)
5323                        .filter(|value| !value.is_empty())
5324                        .map(str::to_string)
5325                });
5326                if child_events
5327                    .iter()
5328                    .filter_map(|event| {
5329                        event
5330                            .payload
5331                            .get("child_workflow_type")
5332                            .or_else(|| event.payload.get("workflow_type"))
5333                            .and_then(Value::as_str)
5334                    })
5335                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
5336                {
5337                    return Err(invalid_recorded_history(
5338                        "child_workflow_identity_mismatch",
5339                        sequence,
5340                        workflow_type
5341                            .as_deref()
5342                            .unwrap_or("one child workflow type"),
5343                        "conflicting child workflow types",
5344                        "child workflow lifecycle events at one sequence disagree on type",
5345                    ));
5346                }
5347                let mut outcomes = child_workflow_outcomes(
5348                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
5349                    fallback_codec,
5350                    parent.clone(),
5351                )?;
5352                if outcomes.len() > 1 {
5353                    return Err(invalid_recorded_history(
5354                        "duplicate_child_workflow_terminal_event",
5355                        sequence,
5356                        "at most one terminal child event",
5357                        "multiple terminal child events",
5358                        "child workflow history settles one command more than once",
5359                    ));
5360                }
5361                return Ok(RecordedCommand::ChildWorkflow {
5362                    sequence,
5363                    workflow_type,
5364                    outcome: outcomes.pop(),
5365                });
5366            }
5367
5368            if !signal_wait_events.is_empty() {
5369                let opened_count = signal_wait_events
5370                    .iter()
5371                    .filter(|event| {
5372                        matches!(
5373                            event.event_type.as_str(),
5374                            "SignalWaitOpened" | "ConditionWaitOpened"
5375                        )
5376                    })
5377                    .count();
5378                if opened_count > 1 {
5379                    return Err(invalid_recorded_history(
5380                        "duplicate_signal_wait_open",
5381                        sequence,
5382                        "at most one signal wait open event",
5383                        "multiple signal wait open events",
5384                        "signal history opens more than one durable wait at one workflow sequence",
5385                    ));
5386                }
5387
5388                let signal_name = signal_wait_events
5389                    .iter()
5390                    .find_map(|event| recorded_signal_wait_name(event));
5391                if signal_wait_events
5392                    .iter()
5393                    .filter_map(|event| recorded_signal_wait_name(event))
5394                    .any(|candidate| Some(candidate.as_str()) != signal_name.as_deref())
5395                {
5396                    return Err(invalid_recorded_history(
5397                        "signal_wait_identity_mismatch",
5398                        sequence,
5399                        signal_name.as_deref().unwrap_or("one signal name"),
5400                        "conflicting signal names",
5401                        "signal wait lifecycle events at one workflow sequence disagree on identity",
5402                    ));
5403                }
5404                return Ok(RecordedCommand::SignalWait {
5405                    sequence,
5406                    signal_name,
5407                });
5408            }
5409
5410            let scheduled: Vec<_> = timer_events
5411                .iter()
5412                .copied()
5413                .filter(|event| event.event_type == "TimerScheduled")
5414                .collect();
5415            let fired: Vec<_> = timer_events
5416                .iter()
5417                .copied()
5418                .filter(|event| event.event_type == "TimerFired")
5419                .collect();
5420            if scheduled.len() != 1 {
5421                return Err(invalid_recorded_history(
5422                    "timer_schedule_missing_or_duplicate",
5423                    sequence,
5424                    "one TimerScheduled event",
5425                    &format!("{} TimerScheduled events", scheduled.len()),
5426                    "timer replay requires exactly one recorded schedule event",
5427                ));
5428            }
5429            if fired.len() > 1 {
5430                return Err(invalid_recorded_history(
5431                    "duplicate_timer_fire",
5432                    sequence,
5433                    "at most one TimerFired event",
5434                    "multiple TimerFired events",
5435                    "timer history contains more than one fire event for a workflow sequence",
5436                ));
5437            }
5438
5439            let scheduled = scheduled[0];
5440            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
5441            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
5442            if let Some(fired) = fired.first() {
5443                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
5444                if fired_timer_id != timer_id {
5445                    return Err(invalid_recorded_history(
5446                        "timer_identity_mismatch",
5447                        sequence,
5448                        &timer_id,
5449                        &fired_timer_id,
5450                        "TimerFired does not correspond to the recorded TimerScheduled event",
5451                    ));
5452                }
5453                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
5454                if fired_delay != delay_seconds {
5455                    return Err(invalid_recorded_history(
5456                        "timer_history_delay_mismatch",
5457                        sequence,
5458                        &delay_seconds.to_string(),
5459                        &fired_delay.to_string(),
5460                        "TimerScheduled and TimerFired record different delays",
5461                    ));
5462                }
5463            }
5464
5465            Ok(RecordedCommand::Timer {
5466                sequence,
5467                delay_seconds,
5468                fired: !fired.is_empty(),
5469            })
5470        })
5471        .collect()
5472}
5473
5474fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
5475    event
5476        .payload
5477        .get("sequence")
5478        .or_else(|| event.payload.get("workflow_sequence"))
5479        .or_else(|| event.raw.get("sequence"))
5480        .or_else(|| event.raw.get("workflow_sequence"))
5481        .and_then(value_as_u64)
5482}
5483
5484fn is_internal_timer_event(event: &HistoryEvent) -> bool {
5485    matches!(
5486        event
5487            .payload
5488            .get("timer_kind")
5489            .or_else(|| event.raw.get("timer_kind"))
5490            .and_then(Value::as_str),
5491        Some("condition_timeout" | "signal_timeout")
5492    )
5493}
5494
5495fn recorded_signal_wait_name(event: &HistoryEvent) -> Option<String> {
5496    match event.event_type.as_str() {
5497        "SignalWaitOpened" | "SignalApplied" => event
5498            .payload
5499            .get("signal_name")
5500            .or_else(|| event.raw.get("signal_name"))
5501            .and_then(Value::as_str)
5502            .filter(|value| !value.is_empty())
5503            .map(str::to_string),
5504        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5505            .payload
5506            .get("condition_key")
5507            .or_else(|| event.raw.get("condition_key"))
5508            .and_then(Value::as_str)
5509            .and_then(|key| key.strip_prefix("signal:"))
5510            .filter(|value| !value.is_empty())
5511            .map(str::to_string),
5512        _ => None,
5513    }
5514}
5515
5516fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
5517    match event.event_type.as_str() {
5518        "SignalWaitOpened" | "SignalApplied" => true,
5519        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5520            .payload
5521            .get("condition_key")
5522            .or_else(|| event.raw.get("condition_key"))
5523            .and_then(Value::as_str)
5524            .is_some_and(|key| key.starts_with("signal:")),
5525        _ => false,
5526    }
5527}
5528
5529fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
5530    event
5531        .payload
5532        .get(field)
5533        .and_then(Value::as_str)
5534        .filter(|value| !value.is_empty())
5535        .map(str::to_string)
5536        .ok_or_else(|| {
5537            invalid_recorded_history(
5538                "timer_history_field_missing",
5539                sequence,
5540                field,
5541                &event.event_type,
5542                "timer history is missing a required identity field",
5543            )
5544        })
5545}
5546
5547fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
5548    event
5549        .payload
5550        .get(field)
5551        .and_then(value_as_u64)
5552        .ok_or_else(|| {
5553            invalid_recorded_history(
5554                "timer_history_field_missing",
5555                sequence,
5556                field,
5557                &event.event_type,
5558                "timer history is missing a required numeric field",
5559            )
5560        })
5561}
5562
5563fn invalid_recorded_history(
5564    reason: &str,
5565    sequence: u64,
5566    expected: &str,
5567    actual: &str,
5568    message: &str,
5569) -> Error {
5570    Error::NonDeterministicReplay(ReplayFailure::new(
5571        reason,
5572        Some(sequence),
5573        Some(expected.to_string()),
5574        Some(actual.to_string()),
5575        message,
5576    ))
5577}
5578
5579type ActivityOutcome = std::result::Result<Value, ActivityFailure>;
5580
5581fn activity_outcome(
5582    event: &HistoryEvent,
5583    fallback_codec: &str,
5584    recorded_activity_type: Option<String>,
5585) -> Result<ActivityOutcome> {
5586    if event.event_type == "ActivityCompleted" {
5587        let codec = event
5588            .payload
5589            .get("payload_codec")
5590            .and_then(Value::as_str)
5591            .unwrap_or(fallback_codec);
5592        return Ok(Ok(decode_wire_value(
5593            event.payload.get("result").unwrap_or(&Value::Null),
5594            codec,
5595        )?));
5596    }
5597
5598    let payload = &event.payload;
5599    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
5600        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
5601        "ActivityCancelled" => (
5602            ActivityFailureKind::Cancelled,
5603            "cancelled",
5604            "activity was cancelled",
5605        ),
5606        "ActivityTimedOut" => (
5607            ActivityFailureKind::TimedOut,
5608            "timeout",
5609            "activity timed out",
5610        ),
5611        _ => unreachable!("activity_outcome is called only for terminal activity events"),
5612    };
5613    let exception = payload
5614        .get("exception")
5615        .filter(|value| !value.is_null())
5616        .cloned();
5617    let failure_category = payload_string(payload, "failure_category");
5618    let timeout_kind = payload_string(payload, "timeout_kind");
5619    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
5620        ActivityFailureKind::Failed => failure_category
5621            .clone()
5622            .unwrap_or_else(|| fallback_reason.to_string()),
5623        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
5624        ActivityFailureKind::TimedOut => timeout_kind
5625            .clone()
5626            .unwrap_or_else(|| fallback_reason.to_string()),
5627    });
5628    let message = payload_string(payload, "message")
5629        .or_else(|| {
5630            exception
5631                .as_ref()
5632                .and_then(|value| payload_string(value, "message"))
5633        })
5634        .unwrap_or_else(|| fallback_message.to_string());
5635
5636    Ok(Err(ActivityFailure {
5637        kind,
5638        reason,
5639        message,
5640        activity_execution_id: payload_string(payload, "activity_execution_id"),
5641        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
5642        activity_type: payload_string(payload, "activity_type")
5643            .or_else(|| payload_string(payload, "activity_name"))
5644            .or(recorded_activity_type),
5645        activity_class: payload_string(payload, "activity_class"),
5646        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
5647        failure_id: payload_string(payload, "failure_id"),
5648        failure_category,
5649        timeout_kind,
5650        non_retryable: payload
5651            .get("non_retryable")
5652            .and_then(Value::as_bool)
5653            .unwrap_or(false),
5654        exception_type: payload_string(payload, "exception_type").or_else(|| {
5655            exception
5656                .as_ref()
5657                .and_then(|value| payload_string(value, "type"))
5658        }),
5659        exception_class: payload_string(payload, "exception_class").or_else(|| {
5660            exception
5661                .as_ref()
5662                .and_then(|value| payload_string(value, "class"))
5663        }),
5664        code: payload
5665            .get("code")
5666            .filter(|value| !value.is_null())
5667            .cloned(),
5668        exception,
5669    }))
5670}
5671
5672type ChildWorkflowOutcome = std::result::Result<ChildWorkflowResult, ChildWorkflowFailure>;
5673
5674fn child_workflow_outcomes(
5675    events: &[HistoryEvent],
5676    fallback_codec: &str,
5677    parent: WorkflowIdentity,
5678) -> Result<Vec<ChildWorkflowOutcome>> {
5679    let mut outcomes = Vec::new();
5680
5681    for event in events {
5682        let kind = match event.event_type.as_str() {
5683            "ChildRunCompleted" => None,
5684            "ChildRunFailed" => Some((
5685                ChildWorkflowFailureKind::Failed,
5686                "child_workflow",
5687                "child workflow failed",
5688            )),
5689            "ChildRunCancelled" => Some((
5690                ChildWorkflowFailureKind::Cancelled,
5691                "cancelled",
5692                "child workflow was cancelled",
5693            )),
5694            "ChildRunTerminated" => Some((
5695                ChildWorkflowFailureKind::Terminated,
5696                "terminated",
5697                "child workflow was terminated",
5698            )),
5699            _ => continue,
5700        };
5701        let payload = &event.payload;
5702        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
5703        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
5704        let child_workflow_type = payload_string(payload, "child_workflow_type");
5705
5706        if let Some((kind, reason, fallback_message)) = kind {
5707            let exception = payload
5708                .get("exception")
5709                .filter(|value| !value.is_null())
5710                .cloned();
5711            let message = payload_string(payload, "message")
5712                .or_else(|| {
5713                    exception
5714                        .as_ref()
5715                        .and_then(|value| payload_string(value, "message"))
5716                })
5717                .unwrap_or_else(|| fallback_message.to_string());
5718            let exception_type = payload_string(payload, "exception_type").or_else(|| {
5719                exception
5720                    .as_ref()
5721                    .and_then(|value| payload_string(value, "type"))
5722            });
5723            let exception_class = payload_string(payload, "exception_class").or_else(|| {
5724                exception
5725                    .as_ref()
5726                    .and_then(|value| payload_string(value, "class"))
5727            });
5728            outcomes.push(Err(ChildWorkflowFailure {
5729                kind,
5730                reason: reason.to_string(),
5731                message,
5732                parent_workflow_id: parent.workflow_id.clone(),
5733                parent_workflow_run_id: parent.run_id.clone(),
5734                child_workflow_id,
5735                child_workflow_run_id,
5736                child_workflow_type,
5737                failure_id: payload_string(payload, "failure_id"),
5738                failure_category: payload_string(payload, "failure_category"),
5739                exception_type,
5740                exception_class,
5741                non_retryable: payload
5742                    .get("non_retryable")
5743                    .and_then(Value::as_bool)
5744                    .unwrap_or(false),
5745                code: payload
5746                    .get("code")
5747                    .filter(|value| !value.is_null())
5748                    .cloned(),
5749                exception,
5750            }));
5751            continue;
5752        }
5753
5754        let codec = payload
5755            .get("payload_codec")
5756            .and_then(Value::as_str)
5757            .unwrap_or(fallback_codec);
5758        let result = payload
5759            .get("result")
5760            .or_else(|| payload.get("output"))
5761            .unwrap_or(&Value::Null);
5762        outcomes.push(Ok(ChildWorkflowResult {
5763            parent: parent.clone(),
5764            child: WorkflowIdentity {
5765                workflow_id: child_workflow_id,
5766                run_id: child_workflow_run_id,
5767            },
5768            child_workflow_type,
5769            result: decode_wire_value(result, codec)?,
5770        }));
5771    }
5772
5773    Ok(outcomes)
5774}
5775
5776fn payload_string(payload: &Value, key: &str) -> Option<String> {
5777    payload
5778        .get(key)
5779        .and_then(Value::as_str)
5780        .filter(|value| !value.is_empty())
5781        .map(str::to_string)
5782}
5783
5784fn workflow_failure_command(error: &Error) -> Value {
5785    let (exception_type, exception_class, properties) = match error {
5786        Error::ActivityFailed(failure) => (
5787            match failure.kind {
5788                ActivityFailureKind::Failed => "ActivityFailed",
5789                ActivityFailureKind::Cancelled => "ActivityCancelled",
5790                ActivityFailureKind::TimedOut => "ActivityTimedOut",
5791            },
5792            "durable_workflow::ActivityFailure",
5793            json!({
5794                "reason": failure.reason,
5795                "activity_execution_id": failure.activity_execution_id,
5796                "activity_attempt_id": failure.activity_attempt_id,
5797                "activity_type": failure.activity_type,
5798                "activity_class": failure.activity_class,
5799                "attempt_number": failure.attempt_number,
5800                "failure_id": failure.failure_id,
5801                "failure_category": failure.failure_category,
5802                "timeout_kind": failure.timeout_kind,
5803                "activity_non_retryable": failure.non_retryable,
5804                "activity_exception_type": failure.exception_type,
5805                "activity_exception_class": failure.exception_class,
5806                "activity_code": failure.code,
5807                "activity_exception": failure.exception,
5808            }),
5809        ),
5810        Error::ChildWorkflowFailed(failure) => (
5811            match failure.kind {
5812                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
5813                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
5814                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
5815            },
5816            "durable_workflow::ChildWorkflowFailure",
5817            json!({
5818                "reason": failure.reason,
5819                "parent_workflow_id": failure.parent_workflow_id,
5820                "parent_workflow_run_id": failure.parent_workflow_run_id,
5821                "child_workflow_id": failure.child_workflow_id,
5822                "child_workflow_run_id": failure.child_workflow_run_id,
5823                "child_workflow_type": failure.child_workflow_type,
5824                "failure_id": failure.failure_id,
5825                "failure_category": failure.failure_category,
5826                "child_exception_type": failure.exception_type,
5827                "child_exception_class": failure.exception_class,
5828                "child_non_retryable": failure.non_retryable,
5829                "child_code": failure.code,
5830                "child_exception": failure.exception,
5831            }),
5832        ),
5833        Error::NonDeterministicReplay(_) => (
5834            "NonDeterministicReplay",
5835            "durable_workflow::Error",
5836            Value::Null,
5837        ),
5838        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
5839    };
5840    let non_retryable = match error {
5841        Error::ActivityFailed(failure) => failure.non_retryable,
5842        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
5843        Error::NonDeterministicReplay(_) => true,
5844        _ => false,
5845    };
5846
5847    json!({
5848        "type": "fail_workflow",
5849        "message": error.to_string(),
5850        "exception_type": exception_type,
5851        "exception_class": exception_class,
5852        "non_retryable": non_retryable,
5853        "exception": {
5854            "type": exception_type,
5855            "class": exception_class,
5856            "message": error.to_string(),
5857            "properties": properties,
5858        }
5859    })
5860}
5861
5862fn signal_values(
5863    events: &[HistoryEvent],
5864    signal_name: &str,
5865    fallback_codec: &str,
5866    resume_signal: Option<&ResumeSignal>,
5867) -> Result<Vec<Vec<Value>>> {
5868    let mut signals = Vec::new();
5869
5870    for event in events {
5871        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
5872            continue;
5873        }
5874
5875        if event.payload.get("signal_name").and_then(Value::as_str) != Some(signal_name) {
5876            continue;
5877        }
5878
5879        let codec = event
5880            .payload
5881            .get("payload_codec")
5882            .and_then(Value::as_str)
5883            .unwrap_or(fallback_codec);
5884        let raw = event
5885            .payload
5886            .get("value")
5887            .or_else(|| event.payload.get("input"))
5888            .or_else(|| event.payload.get("arguments"));
5889        let decoded = match raw.filter(|value| !value.is_null()) {
5890            Some(value) => decode_wire_value(value, codec)?,
5891            None => resume_signal
5892                .filter(|signal| resume_signal_matches_event(signal, event, signal_name))
5893                .map(|signal| Value::Array(signal.arguments.clone()))
5894                .unwrap_or_else(|| Value::Array(Vec::new())),
5895        };
5896        let args = match normalize_arguments(decoded) {
5897            Value::Array(values) => values,
5898            _ => unreachable!("normalize_arguments always returns an array"),
5899        };
5900        signals.push(args);
5901    }
5902
5903    Ok(signals)
5904}
5905
5906fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5907    let Some(export_events) = task
5908        .history_export
5909        .as_ref()
5910        .and_then(|export| export.get("history_events"))
5911        .and_then(Value::as_array)
5912    else {
5913        return Ok(());
5914    };
5915
5916    if export_events.len() > task.history_events.len() {
5917        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
5918    }
5919
5920    Ok(())
5921}
5922
5923fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5924    let Some(export) = task.history_export.as_ref() else {
5925        return Ok(());
5926    };
5927    let signals = export
5928        .get("signals")
5929        .and_then(Value::as_array)
5930        .cloned()
5931        .unwrap_or_default();
5932    let activities = export
5933        .get("activities")
5934        .and_then(Value::as_array)
5935        .cloned()
5936        .unwrap_or_default();
5937    let export_codec = export
5938        .get("payloads")
5939        .and_then(|payloads| payloads.get("codec"))
5940        .and_then(Value::as_str)
5941        .unwrap_or(&task.payload_codec)
5942        .to_string();
5943    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
5944
5945    for event in &mut task.history_events {
5946        if event.event_type == "ActivityCompleted" {
5947            let sequence = event
5948                .payload
5949                .get("sequence")
5950                .or_else(|| event.payload.get("workflow_sequence"))
5951                .and_then(value_as_u64);
5952            let Some(activity) = sequence.and_then(|sequence| {
5953                activities.iter().find(|activity| {
5954                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
5955                })
5956            }) else {
5957                continue;
5958            };
5959            let Some(payload) = event.payload.as_object_mut() else {
5960                continue;
5961            };
5962            if missing_payload(payload.get("result")) {
5963                if let Some(result) = activity
5964                    .get("result")
5965                    .filter(|value| !missing_payload(Some(value)))
5966                {
5967                    payload.insert("result".to_string(), result.clone());
5968                }
5969            }
5970            for field in ["payload_codec", "activity_type"] {
5971                if payload
5972                    .get(field)
5973                    .and_then(Value::as_str)
5974                    .unwrap_or_default()
5975                    .is_empty()
5976                {
5977                    if let Some(value) = activity.get(field) {
5978                        payload.insert(field.to_string(), value.clone());
5979                    }
5980                }
5981            }
5982            continue;
5983        }
5984
5985        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
5986            continue;
5987        }
5988        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
5989        let command_id = event
5990            .payload
5991            .get("workflow_command_id")
5992            .or_else(|| event.raw.get("workflow_command_id"))
5993            .and_then(Value::as_str);
5994        let signal_name = event
5995            .payload
5996            .get("signal_name")
5997            .and_then(Value::as_str)
5998            .unwrap_or_default()
5999            .to_string();
6000        let matched = signals
6001            .iter()
6002            .find(|signal| {
6003                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
6004            })
6005            .or_else(|| {
6006                signals.iter().find(|signal| {
6007                    command_id.is_some()
6008                        && signal.get("command_id").and_then(Value::as_str) == command_id
6009                })
6010            })
6011            .or_else(|| {
6012                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
6013                let signal = signals
6014                    .iter()
6015                    .filter(|signal| {
6016                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
6017                    })
6018                    .nth(*offset);
6019                if signal.is_some() {
6020                    *offset += 1;
6021                }
6022                signal
6023            });
6024        let Some(signal) = matched else {
6025            continue;
6026        };
6027        let signal_codec = signal
6028            .get("payload_codec")
6029            .and_then(Value::as_str)
6030            .unwrap_or(&export_codec);
6031        let Some(payload) = event.payload.as_object_mut() else {
6032            continue;
6033        };
6034        if missing_payload(payload.get("arguments")) {
6035            if let Some(arguments) = signal
6036                .get("arguments")
6037                .filter(|value| !missing_payload(Some(value)))
6038            {
6039                let envelope = match arguments {
6040                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
6041                    other => other.clone(),
6042                };
6043                payload.insert("arguments".to_string(), envelope);
6044            }
6045        }
6046        if payload
6047            .get("payload_codec")
6048            .and_then(Value::as_str)
6049            .unwrap_or_default()
6050            .is_empty()
6051        {
6052            payload.insert("payload_codec".to_string(), json!(signal_codec));
6053        }
6054    }
6055
6056    Ok(())
6057}
6058
6059fn missing_payload(value: Option<&Value>) -> bool {
6060    match value {
6061        None | Some(Value::Null) => true,
6062        Some(Value::String(value)) => value.is_empty(),
6063        Some(_) => false,
6064    }
6065}
6066
6067fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
6068    let export_signals = task
6069        .history_export
6070        .as_ref()
6071        .and_then(|export| export.get("signals"))
6072        .and_then(Value::as_array)
6073        .cloned()
6074        .unwrap_or_default();
6075    let export_codec = task
6076        .history_export
6077        .as_ref()
6078        .and_then(|export| export.get("payloads"))
6079        .and_then(|payloads| payloads.get("codec"))
6080        .and_then(Value::as_str)
6081        .unwrap_or(&task.payload_codec);
6082    let mut name_offsets: HashMap<String, usize> = HashMap::new();
6083    let mut signals = Vec::new();
6084
6085    for event in &task.history_events {
6086        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
6087            continue;
6088        }
6089
6090        let name = event
6091            .payload
6092            .get("signal_name")
6093            .and_then(Value::as_str)
6094            .unwrap_or_default();
6095        if name.is_empty() {
6096            continue;
6097        }
6098        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
6099        let command_id = event
6100            .payload
6101            .get("workflow_command_id")
6102            .or_else(|| event.raw.get("workflow_command_id"))
6103            .and_then(Value::as_str);
6104        let matched_export = export_signals
6105            .iter()
6106            .find(|candidate| {
6107                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
6108            })
6109            .or_else(|| {
6110                export_signals.iter().find(|candidate| {
6111                    command_id.is_some()
6112                        && candidate.get("command_id").and_then(Value::as_str) == command_id
6113                })
6114            })
6115            .or_else(|| {
6116                let offset = name_offsets.entry(name.to_string()).or_default();
6117                let candidate = export_signals
6118                    .iter()
6119                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
6120                    .nth(*offset);
6121                if candidate.is_some() {
6122                    *offset += 1;
6123                }
6124                candidate
6125            });
6126        let codec = event
6127            .payload
6128            .get("payload_codec")
6129            .and_then(Value::as_str)
6130            .or_else(|| {
6131                matched_export
6132                    .and_then(|signal| signal.get("payload_codec"))
6133                    .and_then(Value::as_str)
6134            })
6135            .unwrap_or(export_codec);
6136        let raw_arguments = event
6137            .payload
6138            .get("value")
6139            .or_else(|| event.payload.get("input"))
6140            .or_else(|| event.payload.get("arguments"))
6141            .filter(|value| !value.is_null())
6142            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
6143        let arguments = decode_query_signal_arguments(raw_arguments, codec)?;
6144        let workflow_sequence = event
6145            .payload
6146            .get("workflow_sequence")
6147            .and_then(value_as_u64)
6148            .or_else(|| {
6149                matched_export
6150                    .and_then(|signal| signal.get("workflow_sequence"))
6151                    .and_then(value_as_u64)
6152            });
6153
6154        signals.push(QuerySignal {
6155            id: signal_id.map(str::to_string).or_else(|| {
6156                matched_export
6157                    .and_then(|signal| signal.get("id"))
6158                    .and_then(Value::as_str)
6159                    .map(str::to_string)
6160            }),
6161            name: name.to_string(),
6162            arguments,
6163            workflow_sequence,
6164        });
6165    }
6166
6167    if signals.is_empty() {
6168        for signal in export_signals {
6169            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
6170                continue;
6171            }
6172            let Some(name) = signal.get("name").and_then(Value::as_str) else {
6173                continue;
6174            };
6175            let codec = signal
6176                .get("payload_codec")
6177                .and_then(Value::as_str)
6178                .unwrap_or(export_codec);
6179            let arguments = decode_query_signal_arguments(signal.get("arguments"), codec)?;
6180            signals.push(QuerySignal {
6181                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
6182                name: name.to_string(),
6183                arguments,
6184                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
6185            });
6186        }
6187        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
6188    }
6189
6190    Ok(signals)
6191}
6192
6193fn decode_query_signal_arguments(raw: Option<&Value>, codec: &str) -> Result<Vec<Value>> {
6194    let decoded = match raw.filter(|value| !value.is_null()) {
6195        Some(value) => decode_wire_value(value, codec)?,
6196        None => Value::Array(Vec::new()),
6197    };
6198    let Value::Array(arguments) = normalize_arguments(decoded) else {
6199        unreachable!("normalize_arguments always returns an array");
6200    };
6201    Ok(arguments)
6202}
6203
6204fn value_as_u64(value: &Value) -> Option<u64> {
6205    value
6206        .as_u64()
6207        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
6208}
6209
6210fn resume_signal_matches_event(
6211    resume_signal: &ResumeSignal,
6212    event: &HistoryEvent,
6213    signal_name: &str,
6214) -> bool {
6215    if resume_signal.signal_name != signal_name {
6216        return false;
6217    }
6218
6219    match (
6220        resume_signal.signal_id.as_deref(),
6221        event.payload.get("signal_id").and_then(Value::as_str),
6222    ) {
6223        (Some(resume_id), Some(event_id)) => resume_id == event_id,
6224        _ => true,
6225    }
6226}
6227
6228#[cfg(test)]
6229mod tests {
6230    use super::*;
6231    use std::{
6232        io::{Read, Write},
6233        net::{SocketAddr, TcpListener, TcpStream},
6234        thread,
6235    };
6236
6237    #[derive(Clone, Debug, Default, PartialEq)]
6238    struct ReplayCounterState {
6239        loaded: Option<String>,
6240        count: i64,
6241        finished: bool,
6242    }
6243
6244    fn replay_counter_worker() -> Worker {
6245        let client = Client::new("http://127.0.0.1:8080").expect("client");
6246        let mut worker = Worker::new(client, "rust-workers");
6247        worker.register_replayed_workflow(
6248            "replay-counter",
6249            ReplayCounterState::default,
6250            |ctx, _input, state| async move {
6251                let loaded = ctx.activity("load-counter", json!([])).await?;
6252                state.update(|current| {
6253                    current.loaded = loaded.as_str().map(str::to_string);
6254                })?;
6255                for _ in 0..2 {
6256                    let signal = ctx.wait_signal("increment").await?;
6257                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
6258                    state.update(|current| current.count += amount)?;
6259                }
6260                state.update(|current| current.finished = true)?;
6261                state.read(|current| Ok(json!(current.count)))?
6262            },
6263        );
6264        worker.register_replayed_query::<ReplayCounterState, _, _>(
6265            "replay-counter",
6266            "current",
6267            |_ctx, state, _args| async move {
6268                Ok(json!({
6269                    "loaded": state.loaded,
6270                    "count": state.count,
6271                    "finished": state.finished,
6272                }))
6273            },
6274        );
6275        worker.register_replayed_query::<ReplayCounterState, _, _>(
6276            "replay-counter",
6277            "detached-mutation",
6278            |_ctx, state, _args| async move {
6279                let mut detached = (*state).clone();
6280                detached.count = 999;
6281                Ok(json!(detached.count))
6282            },
6283        );
6284        worker.register_replayed_query::<ReplayCounterState, _, _>(
6285            "replay-counter",
6286            "failed-mutation",
6287            |_ctx, state, _args| async move {
6288                let mut detached = (*state).clone();
6289                detached.count = 999;
6290                Err(Error::WorkerLoop("query refused".to_string()))
6291            },
6292        );
6293        worker
6294    }
6295
6296    fn replay_counter_query(
6297        query_name: &str,
6298        history_events: Value,
6299        run_status: &str,
6300    ) -> QueryTask {
6301        serde_json::from_value(json!({
6302            "query_task_id": format!("query-{query_name}"),
6303            "workflow_type": "replay-counter",
6304            "query_name": query_name,
6305            "payload_codec": "json",
6306            "workflow_arguments": {"codec": "json", "blob": "[]"},
6307            "query_arguments": {"codec": "json", "blob": "[]"},
6308            "history_events": history_events,
6309            "run_status": run_status,
6310        }))
6311        .expect("query task")
6312    }
6313
6314    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
6315        WorkflowContext {
6316            state: Arc::new(Mutex::new(
6317                WorkflowState::new_with_identity(
6318                    history,
6319                    None,
6320                    None,
6321                    "rust-workers".to_string(),
6322                    JSON_CODEC.to_string(),
6323                    None,
6324                )
6325                .expect("valid workflow history"),
6326            )),
6327        }
6328    }
6329
6330    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
6331        HistoryEvent {
6332            event_type: event_type.to_string(),
6333            payload,
6334            raw: HashMap::new(),
6335        }
6336    }
6337
6338    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
6339        vec![
6340            history_event(
6341                "ActivityScheduled",
6342                json!({
6343                    "sequence": 1,
6344                    "activity_type": "flaky",
6345                    "activity_execution_id": "act-1",
6346                    "activity": {
6347                        "id": "act-1",
6348                        "sequence": 1,
6349                        "type": "flaky",
6350                        "queue": "critical-activities",
6351                        "execution_mode": null,
6352                        "retry_policy": {
6353                            "snapshot_version": 1,
6354                            "max_attempts": 3,
6355                            "backoff_seconds": [2, 4],
6356                            "start_to_close_timeout": 30,
6357                            "schedule_to_start_timeout": 5,
6358                            "schedule_to_close_timeout": 90,
6359                            "heartbeat_timeout": 10,
6360                            "non_retryable_error_types": ["PermanentError"]
6361                        }
6362                    }
6363                }),
6364            ),
6365            history_event(
6366                "ActivityStarted",
6367                json!({
6368                    "sequence": 1,
6369                    "activity_type": "flaky",
6370                    "activity_execution_id": "act-1",
6371                    "activity_attempt_id": "attempt-1",
6372                    "attempt_number": 1
6373                }),
6374            ),
6375            history_event(
6376                "ActivityRetryScheduled",
6377                json!({
6378                    "sequence": 1,
6379                    "activity_type": "flaky",
6380                    "activity_execution_id": "act-1",
6381                    "activity_attempt_id": "attempt-1",
6382                    "attempt_number": 1,
6383                    "retry_after_attempt": 1,
6384                    "retry_backoff_seconds": 2,
6385                    "failure_category": "activity",
6386                    "exception_type": "TransientError"
6387                }),
6388            ),
6389            history_event(
6390                "ActivityStarted",
6391                json!({
6392                    "sequence": 1,
6393                    "activity_type": "flaky",
6394                    "activity_execution_id": "act-1",
6395                    "activity_attempt_id": "attempt-2",
6396                    "attempt_number": 2
6397                }),
6398            ),
6399            history_event(
6400                "ActivityCompleted",
6401                json!({
6402                    "sequence": 1,
6403                    "activity_type": "flaky",
6404                    "activity_execution_id": "act-1",
6405                    "activity_attempt_id": "attempt-2",
6406                    "attempt_number": 2,
6407                    "payload_codec": "json",
6408                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
6409                }),
6410            ),
6411        ]
6412    }
6413
6414    fn retry_activity_options() -> ActivityOptions {
6415        ActivityOptions::new()
6416            .task_queue("critical-activities")
6417            .retry_policy(
6418                ActivityRetryPolicy::new(3)
6419                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
6420                    .non_retryable_error_type("PermanentError"),
6421            )
6422            .start_to_close_timeout(Duration::from_secs(30))
6423            .schedule_to_start_timeout(Duration::from_secs(5))
6424            .schedule_to_close_timeout(Duration::from_secs(90))
6425            .heartbeat_timeout(Duration::from_secs(10))
6426    }
6427
6428    #[test]
6429    fn avro_generic_wrapper_round_trips_json_values() {
6430        let value = json!({"greeting": "hello", "count": 3, "ok": true});
6431        let envelope = PayloadEnvelope::avro(&value).expect("encode");
6432        assert_eq!(envelope.codec, DEFAULT_CODEC);
6433        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6434    }
6435
6436    #[test]
6437    fn json_codec_remains_plain_json() {
6438        let value = json!({"greeting": "hello", "count": 3, "ok": true});
6439        let envelope = PayloadEnvelope::json(&value).expect("encode");
6440
6441        assert_eq!(envelope.codec, JSON_CODEC);
6442        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
6443        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6444    }
6445
6446    #[test]
6447    fn typed_avro_payload_without_schema_context_keeps_diagnostic() {
6448        let envelope = PayloadEnvelope {
6449            codec: DEFAULT_CODEC.to_string(),
6450            blob: BASE64.encode([0x01]),
6451        };
6452
6453        let error = decode_payload::<Value>(&envelope).expect_err("typed payload must fail");
6454        assert_eq!(
6455            error.to_string(),
6456            "codec error: typed avro payloads require a schema context; v1 supports the generic wrapper"
6457        );
6458    }
6459
6460    #[test]
6461    fn workflow_context_schedules_activity_until_completion_is_in_history() {
6462        let ctx = WorkflowContext {
6463            state: Arc::new(Mutex::new(
6464                WorkflowState::new_with_identity(
6465                    Vec::new(),
6466                    Some("wf-parent".to_string()),
6467                    Some("run-parent".to_string()),
6468                    "rust-workers".to_string(),
6469                    DEFAULT_CODEC.to_string(),
6470                    None,
6471                )
6472                .expect("workflow state"),
6473            )),
6474        };
6475
6476        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
6477        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6478        assert!(matches!(
6479            call.as_mut().poll(&mut task_context),
6480            Poll::Pending
6481        ));
6482
6483        let commands = ctx.take_commands().expect("commands");
6484        assert_eq!(commands[0]["type"], "schedule_activity");
6485        assert_eq!(commands[0]["activity_type"], "hello.activity");
6486    }
6487
6488    #[test]
6489    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
6490        let ctx = workflow_context(Vec::new());
6491        let options = ActivityOptions::new()
6492            .task_queue("payments")
6493            .retry_policy(
6494                ActivityRetryPolicy::new(4)
6495                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
6496                    .non_retryable_error_type("ValidationError"),
6497            )
6498            .start_to_close_timeout(Duration::from_secs(120))
6499            .schedule_to_start_timeout(Duration::from_secs(10))
6500            .schedule_to_close_timeout(Duration::from_secs(300))
6501            .heartbeat_timeout(Duration::from_secs(15));
6502        let mut call = Box::pin(ctx.activity_with_options(
6503            "charge-card",
6504            options,
6505            json!([{"order_id": "o-1"}]),
6506        ));
6507        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6508
6509        assert!(matches!(
6510            call.as_mut().poll(&mut task_context),
6511            Poll::Pending
6512        ));
6513        assert!(matches!(
6514            call.as_mut().poll(&mut task_context),
6515            Poll::Pending
6516        ));
6517
6518        let commands = ctx.take_commands().expect("activity command");
6519        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
6520        assert_eq!(commands[0]["queue"], "payments");
6521        assert_eq!(
6522            commands[0]["retry_policy"],
6523            json!({
6524                "max_attempts": 4,
6525                "backoff_seconds": [1, 3, 9],
6526                "non_retryable_error_types": ["ValidationError"],
6527            })
6528        );
6529        assert_eq!(commands[0]["start_to_close_timeout"], 120);
6530        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
6531        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
6532        assert_eq!(commands[0]["heartbeat_timeout"], 15);
6533    }
6534
6535    #[test]
6536    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
6537        let ctx = workflow_context(Vec::new());
6538        let options = ActivityOptions::new().retry_policy(
6539            ActivityRetryPolicy::new(3)
6540                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
6541        );
6542        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6543        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6544
6545        assert!(matches!(
6546            call.as_mut().poll(&mut task_context),
6547            Poll::Pending
6548        ));
6549        assert_eq!(
6550            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
6551            json!([1, 2])
6552        );
6553    }
6554
6555    #[test]
6556    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
6557        let cases = [
6558            (
6559                ActivityOptions::new().task_queue("  "),
6560                ActivityOptionsErrorKind::EmptyTaskQueue,
6561            ),
6562            (
6563                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
6564                ActivityOptionsErrorKind::EmptyRetryPolicy,
6565            ),
6566            (
6567                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
6568                ActivityOptionsErrorKind::InvalidMaxAttempts,
6569            ),
6570            (
6571                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
6572                    max_attempts: None,
6573                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
6574                    non_retryable_error_types: Vec::new(),
6575                }),
6576                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
6577            ),
6578            (
6579                ActivityOptions::new().retry_policy(
6580                    ActivityRetryPolicy::new(2)
6581                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
6582                ),
6583                ActivityOptionsErrorKind::TooManyBackoffIntervals,
6584            ),
6585            (
6586                ActivityOptions::new().retry_policy(
6587                    ActivityRetryPolicy::new(2).exponential_backoff(
6588                        Duration::from_secs(1),
6589                        0,
6590                        None,
6591                    ),
6592                ),
6593                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
6594            ),
6595            (
6596                ActivityOptions::new()
6597                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
6598                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
6599            ),
6600            (
6601                ActivityOptions::new().retry_policy(
6602                    ActivityRetryPolicy::new(10_002).exponential_backoff(
6603                        Duration::from_secs(1),
6604                        1,
6605                        None,
6606                    ),
6607                ),
6608                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
6609            ),
6610            (
6611                ActivityOptions::new().retry_policy(
6612                    ActivityRetryPolicy::new(2)
6613                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
6614                ),
6615                ActivityOptionsErrorKind::BackoffOverflow,
6616            ),
6617        ];
6618
6619        for (options, expected_kind) in cases {
6620            let ctx = workflow_context(Vec::new());
6621            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6622            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6623            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
6624                call.as_mut().poll(&mut task_context)
6625            else {
6626                panic!("expected typed activity validation error");
6627            };
6628            assert_eq!(error.kind, expected_kind);
6629            assert!(ctx.take_commands().expect("commands").is_empty());
6630        }
6631    }
6632
6633    #[test]
6634    fn activity_options_validate_positive_and_ordered_timeouts() {
6635        let zero_timeout_cases = [
6636            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
6637            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
6638            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
6639            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
6640        ];
6641        for options in zero_timeout_cases {
6642            assert_eq!(
6643                options.validate().expect_err("zero timeout").kind,
6644                ActivityOptionsErrorKind::TimeoutNotPositive
6645            );
6646        }
6647
6648        let ordering_cases = [
6649            ActivityOptions::new()
6650                .heartbeat_timeout(Duration::from_secs(11))
6651                .start_to_close_timeout(Duration::from_secs(10)),
6652            ActivityOptions::new()
6653                .start_to_close_timeout(Duration::from_secs(31))
6654                .schedule_to_close_timeout(Duration::from_secs(30)),
6655            ActivityOptions::new()
6656                .schedule_to_start_timeout(Duration::from_secs(31))
6657                .schedule_to_close_timeout(Duration::from_secs(30)),
6658        ];
6659        for options in ordering_cases {
6660            assert_eq!(
6661                options.validate().expect_err("timeout order").kind,
6662                ActivityOptionsErrorKind::TimeoutOrder
6663            );
6664        }
6665
6666        assert_eq!(
6667            ActivityOptions::new()
6668                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
6669                .validate()
6670                .expect_err("protocol integer overflow")
6671                .kind,
6672            ActivityOptionsErrorKind::TimeoutOverflow
6673        );
6674    }
6675
6676    #[test]
6677    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
6678        let ctx = workflow_context(completed_retry_activity_history());
6679        let mut call =
6680            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6681        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6682
6683        assert!(matches!(
6684            call.as_mut().poll(&mut task_context),
6685            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6686        ));
6687        assert!(ctx.take_commands().expect("commands").is_empty());
6688        ctx.ensure_history_consumed().expect("history consumed");
6689    }
6690
6691    #[test]
6692    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
6693        let mut options = retry_activity_options();
6694        options
6695            .retry_policy
6696            .as_mut()
6697            .expect("retry policy")
6698            .non_retryable_error_types
6699            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
6700
6701        let new_ctx = workflow_context(Vec::new());
6702        let mut new_call =
6703            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
6704        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6705        assert!(matches!(
6706            new_call.as_mut().poll(&mut task_context),
6707            Poll::Pending
6708        ));
6709        let commands = new_ctx.take_commands().expect("commands");
6710        assert_eq!(commands.len(), 1);
6711        assert_eq!(
6712            commands[0]["retry_policy"]["non_retryable_error_types"],
6713            json!(["PermanentError"])
6714        );
6715
6716        let replay_ctx = workflow_context(completed_retry_activity_history());
6717        let mut replay_call =
6718            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
6719        assert!(matches!(
6720            replay_call.as_mut().poll(&mut task_context),
6721            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6722        ));
6723        assert!(replay_ctx.take_commands().expect("commands").is_empty());
6724        replay_ctx
6725            .ensure_history_consumed()
6726            .expect("history consumed");
6727    }
6728
6729    #[test]
6730    fn replayed_intermediate_retry_remains_pending_across_restarts() {
6731        let history = completed_retry_activity_history()
6732            .into_iter()
6733            .take(3)
6734            .collect::<Vec<_>>();
6735
6736        for _restart in 0..2 {
6737            let ctx = workflow_context(history.clone());
6738            let mut call =
6739                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6740            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6741            assert!(matches!(
6742                call.as_mut().poll(&mut task_context),
6743                Poll::Pending
6744            ));
6745            assert!(ctx.take_commands().expect("commands").is_empty());
6746        }
6747    }
6748
6749    #[test]
6750    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
6751        let mut changed_queue = retry_activity_options();
6752        changed_queue.task_queue = Some("different-queue".to_string());
6753
6754        let mut changed_max_attempts = retry_activity_options();
6755        let retry_policy = changed_max_attempts
6756            .retry_policy
6757            .as_mut()
6758            .expect("retry policy");
6759        retry_policy.max_attempts = Some(4);
6760
6761        let mut changed_backoff = retry_activity_options();
6762        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
6763        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
6764            Duration::from_secs(3),
6765            Duration::from_secs(4),
6766        ]));
6767
6768        let mut changed_non_retryable_types = retry_activity_options();
6769        let retry_policy = changed_non_retryable_types
6770            .retry_policy
6771            .as_mut()
6772            .expect("retry policy");
6773        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
6774
6775        let mut changed_start_to_close = retry_activity_options();
6776        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
6777        let mut changed_schedule_to_start = retry_activity_options();
6778        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
6779        let mut changed_schedule_to_close = retry_activity_options();
6780        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
6781        let mut changed_heartbeat = retry_activity_options();
6782        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
6783
6784        let cases = [
6785            (changed_queue, "activity_task_queue_mismatch"),
6786            (changed_max_attempts, "activity_retry_policy_mismatch"),
6787            (changed_backoff, "activity_retry_policy_mismatch"),
6788            (
6789                changed_non_retryable_types,
6790                "activity_retry_policy_mismatch",
6791            ),
6792            (changed_start_to_close, "activity_retry_policy_mismatch"),
6793            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
6794            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
6795            (changed_heartbeat, "activity_retry_policy_mismatch"),
6796        ];
6797
6798        for (options, expected_reason) in cases {
6799            let ctx = workflow_context(completed_retry_activity_history());
6800            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
6801            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6802            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6803                call.as_mut().poll(&mut task_context)
6804            else {
6805                panic!("changed activity options must fail replay");
6806            };
6807            assert_eq!(failure.reason, expected_reason);
6808            assert_eq!(failure.sequence, Some(1));
6809            assert!(ctx.take_commands().expect("commands").is_empty());
6810        }
6811    }
6812
6813    #[test]
6814    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
6815        let cases = [
6816            (
6817                "execution_mode",
6818                json!("local"),
6819                "activity_execution_mode_mismatch",
6820            ),
6821            (
6822                "snapshot_version",
6823                json!(2),
6824                "activity_retry_policy_mismatch",
6825            ),
6826        ];
6827
6828        for (field, value, expected_reason) in cases {
6829            let mut history = completed_retry_activity_history();
6830            let activity = history[0].payload["activity"]
6831                .as_object_mut()
6832                .expect("activity snapshot");
6833            if field == "execution_mode" {
6834                activity.insert(field.to_string(), value);
6835            } else {
6836                activity["retry_policy"]
6837                    .as_object_mut()
6838                    .expect("retry snapshot")
6839                    .insert(field.to_string(), value);
6840            }
6841
6842            let ctx = workflow_context(history);
6843            let mut call =
6844                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6845            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6846            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6847                call.as_mut().poll(&mut task_context)
6848            else {
6849                panic!("changed {field} must fail replay");
6850            };
6851            assert_eq!(failure.reason, expected_reason);
6852            assert_eq!(failure.sequence, Some(1));
6853            assert!(ctx.take_commands().expect("commands").is_empty());
6854        }
6855    }
6856
6857    #[test]
6858    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
6859        let mut history = completed_retry_activity_history();
6860        let activity = history[0].payload["activity"]
6861            .as_object_mut()
6862            .expect("activity snapshot");
6863        activity.remove("execution_mode");
6864        activity.remove("retry_policy");
6865
6866        let mut current = retry_activity_options();
6867        current.start_to_close_timeout = Some(Duration::from_secs(45));
6868        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
6869        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
6870        current.heartbeat_timeout = Some(Duration::from_secs(12));
6871
6872        let ctx = workflow_context(history);
6873        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
6874        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6875        assert!(matches!(
6876            call.as_mut().poll(&mut task_context),
6877            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6878        ));
6879        assert!(ctx.take_commands().expect("commands").is_empty());
6880        ctx.ensure_history_consumed().expect("history consumed");
6881    }
6882
6883    #[test]
6884    fn terminal_activity_failed_after_start_returns_typed_failure() {
6885        let history = vec![
6886            history_event(
6887                "ActivityScheduled",
6888                json!({
6889                    "sequence": 1,
6890                    "activity_type": "flaky",
6891                    "activity_execution_id": "act-terminal",
6892                    "activity": {
6893                        "id": "act-terminal",
6894                        "sequence": 1,
6895                        "type": "flaky",
6896                        "queue": "critical-activities",
6897                        "retry_policy": {
6898                            "snapshot_version": 1,
6899                            "max_attempts": 3,
6900                            "backoff_seconds": [2, 4],
6901                            "non_retryable_error_types": ["PermanentError"]
6902                        }
6903                    }
6904                }),
6905            ),
6906            history_event(
6907                "ActivityStarted",
6908                json!({
6909                    "sequence": 1,
6910                    "activity_type": "flaky",
6911                    "activity_execution_id": "act-terminal",
6912                    "activity_attempt_id": "attempt-1",
6913                    "attempt_number": 1
6914                }),
6915            ),
6916            history_event(
6917                "ActivityFailed",
6918                json!({
6919                    "sequence": 1,
6920                    "activity_type": "flaky",
6921                    "activity_execution_id": "act-terminal",
6922                    "activity_attempt_id": "attempt-1",
6923                    "attempt_number": 1,
6924                    "failure_id": "failure-terminal",
6925                    "failure_category": "activity",
6926                    "exception_type": "PermanentError",
6927                    "message": "cannot retry",
6928                    "non_retryable": true
6929                }),
6930            ),
6931        ];
6932        let ctx = workflow_context(history);
6933        let mut call =
6934            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6935        let mut task_context = TaskContext::from_waker(noop_waker_ref());
6936
6937        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6938            call.as_mut().poll(&mut task_context)
6939        else {
6940            panic!("terminal ActivityFailed must settle the activity future");
6941        };
6942        assert_eq!(failure.kind, ActivityFailureKind::Failed);
6943        assert_eq!(
6944            failure.activity_execution_id.as_deref(),
6945            Some("act-terminal")
6946        );
6947        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
6948        assert!(failure.non_retryable);
6949        assert!(ctx.take_commands().expect("commands").is_empty());
6950        ctx.ensure_history_consumed().expect("history consumed");
6951    }
6952
6953    #[test]
6954    fn activity_terminal_events_return_machine_readable_failures() {
6955        let cases = [
6956            (
6957                "ActivityFailed",
6958                json!({
6959                    "sequence": 1,
6960                    "activity_type": "charge-card",
6961                    "activity_execution_id": "act-1",
6962                    "activity_attempt_id": "attempt-2",
6963                    "attempt_number": 2,
6964                    "failure_id": "failure-1",
6965                    "failure_category": "activity",
6966                    "exception_type": "PaymentDeclined",
6967                    "exception_class": "payments.PaymentDeclined",
6968                    "message": "card declined",
6969                    "non_retryable": true
6970                }),
6971                ActivityFailureKind::Failed,
6972                "activity",
6973            ),
6974            (
6975                "ActivityCancelled",
6976                json!({
6977                    "sequence": 1,
6978                    "activity_type": "charge-card",
6979                    "activity_execution_id": "act-1",
6980                    "activity_attempt_id": "attempt-1"
6981                }),
6982                ActivityFailureKind::Cancelled,
6983                "cancelled",
6984            ),
6985        ];
6986
6987        for (event_type, payload, expected_kind, expected_reason) in cases {
6988            let ctx = workflow_context(vec![history_event(event_type, payload)]);
6989            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
6990            let mut task_context = TaskContext::from_waker(noop_waker_ref());
6991            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6992                call.as_mut().poll(&mut task_context)
6993            else {
6994                panic!("expected terminal activity failure");
6995            };
6996            assert_eq!(failure.kind, expected_kind);
6997            assert_eq!(failure.reason, expected_reason);
6998            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
6999            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
7000        }
7001    }
7002
7003    #[test]
7004    fn every_activity_timeout_class_is_typed() {
7005        for timeout_kind in [
7006            "start_to_close",
7007            "schedule_to_start",
7008            "schedule_to_close",
7009            "heartbeat",
7010        ] {
7011            let ctx = workflow_context(vec![history_event(
7012                "ActivityTimedOut",
7013                json!({
7014                    "sequence": 1,
7015                    "activity_type": "slow",
7016                    "activity_execution_id": "act-timeout",
7017                    "activity_attempt_id": "attempt-timeout",
7018                    "failure_category": "timeout",
7019                    "timeout_kind": timeout_kind,
7020                    "message": "deadline expired"
7021                }),
7022            )]);
7023            let mut call = Box::pin(ctx.activity("slow", json!([])));
7024            let mut task_context = TaskContext::from_waker(noop_waker_ref());
7025            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
7026                call.as_mut().poll(&mut task_context)
7027            else {
7028                panic!("expected timeout failure");
7029            };
7030            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
7031            assert_eq!(failure.reason, timeout_kind);
7032            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
7033            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
7034        }
7035    }
7036
7037    #[test]
7038    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
7039        let ctx = workflow_context(Vec::new());
7040        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
7041        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7042
7043        assert!(matches!(
7044            sleep.as_mut().poll(&mut task_context),
7045            Poll::Pending
7046        ));
7047        assert!(matches!(
7048            sleep.as_mut().poll(&mut task_context),
7049            Poll::Pending
7050        ));
7051
7052        let commands = ctx.take_commands().expect("timer command");
7053        assert_eq!(
7054            commands,
7055            vec![json!({
7056                "type": "start_timer",
7057                "delay_seconds": 2,
7058            })]
7059        );
7060    }
7061
7062    #[test]
7063    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
7064        let history = vec![
7065            history_event(
7066                "TimerScheduled",
7067                json!({
7068                    "sequence": 1,
7069                    "timer_id": "timer-1",
7070                    "delay_seconds": 5,
7071                    "fire_at": "2026-07-11T12:00:05Z",
7072                }),
7073            ),
7074            history_event(
7075                "TimerFired",
7076                json!({
7077                    "sequence": 1,
7078                    "timer_id": "timer-1",
7079                    "delay_seconds": 5,
7080                    "fire_at": "2026-07-11T12:00:05Z",
7081                    "fired_at": "2026-07-11T12:00:05Z",
7082                }),
7083            ),
7084        ];
7085
7086        for _restart in 0..2 {
7087            let ctx = workflow_context(history.clone());
7088            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
7089            let mut task_context = TaskContext::from_waker(noop_waker_ref());
7090            assert!(matches!(
7091                sleep.as_mut().poll(&mut task_context),
7092                Poll::Ready(Ok(()))
7093            ));
7094            assert!(ctx.take_commands().expect("commands").is_empty());
7095            ctx.ensure_history_consumed().expect("history consumed");
7096        }
7097    }
7098
7099    #[test]
7100    fn workflow_sleep_rejects_changed_delay_during_replay() {
7101        let ctx = workflow_context(vec![
7102            history_event(
7103                "TimerScheduled",
7104                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7105            ),
7106            history_event(
7107                "TimerFired",
7108                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7109            ),
7110        ]);
7111        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
7112        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7113
7114        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7115            sleep.as_mut().poll(&mut task_context)
7116        else {
7117            panic!("changed timer delay must be rejected");
7118        };
7119        assert_eq!(failure.reason, "timer_delay_mismatch");
7120        assert_eq!(failure.sequence, Some(1));
7121    }
7122
7123    #[test]
7124    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
7125        let lone_fire = WorkflowState::new(
7126            vec![history_event(
7127                "TimerFired",
7128                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7129            )],
7130            "rust-workers".to_string(),
7131            JSON_CODEC.to_string(),
7132            None,
7133        )
7134        .expect_err("TimerFired requires TimerScheduled");
7135        assert!(matches!(
7136            lone_fire,
7137            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7138                if reason == "timer_schedule_missing_or_duplicate"
7139        ));
7140
7141        let wrong_identity = WorkflowState::new(
7142            vec![
7143                history_event(
7144                    "TimerScheduled",
7145                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7146                ),
7147                history_event(
7148                    "TimerFired",
7149                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7150                ),
7151            ],
7152            "rust-workers".to_string(),
7153            JSON_CODEC.to_string(),
7154            None,
7155        )
7156        .expect_err("fire must match scheduled timer identity");
7157        assert!(matches!(
7158            wrong_identity,
7159            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7160                if reason == "timer_identity_mismatch"
7161        ));
7162
7163        let duplicate_fire = WorkflowState::new(
7164            vec![
7165                history_event(
7166                    "TimerScheduled",
7167                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7168                ),
7169                history_event(
7170                    "TimerFired",
7171                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7172                ),
7173                history_event(
7174                    "TimerFired",
7175                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7176                ),
7177            ],
7178            "rust-workers".to_string(),
7179            JSON_CODEC.to_string(),
7180            None,
7181        )
7182        .expect_err("a durable timer cannot fire twice");
7183        assert!(matches!(
7184            duplicate_fire,
7185            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7186                if reason == "duplicate_timer_fire"
7187        ));
7188
7189        let wrong_fired_delay = WorkflowState::new(
7190            vec![
7191                history_event(
7192                    "TimerScheduled",
7193                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7194                ),
7195                history_event(
7196                    "TimerFired",
7197                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
7198                ),
7199            ],
7200            "rust-workers".to_string(),
7201            JSON_CODEC.to_string(),
7202            None,
7203        )
7204        .expect_err("timer schedule and fire delays must agree");
7205        assert!(matches!(
7206            wrong_fired_delay,
7207            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7208                if reason == "timer_history_delay_mismatch"
7209        ));
7210    }
7211
7212    #[test]
7213    fn replay_rejects_activity_moved_before_recorded_timer() {
7214        let ctx = workflow_context(vec![
7215            history_event(
7216                "TimerScheduled",
7217                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7218            ),
7219            history_event(
7220                "TimerFired",
7221                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7222            ),
7223            history_event(
7224                "ActivityCompleted",
7225                json!({
7226                    "sequence": 2,
7227                    "activity_type": "after-timer",
7228                    "payload_codec": "json",
7229                    "result": {"codec": "json", "blob": "\"done\""},
7230                }),
7231            ),
7232        ]);
7233        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
7234        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7235
7236        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7237            activity.as_mut().poll(&mut task_context)
7238        else {
7239            panic!("reordered durable command must be rejected");
7240        };
7241        assert_eq!(failure.reason, "recorded_command_mismatch");
7242        assert_eq!(failure.sequence, Some(1));
7243        assert_eq!(failure.expected.as_deref(), Some("timer"));
7244        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
7245    }
7246
7247    #[test]
7248    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
7249        let signal_then_timer = vec![
7250            history_event(
7251                "ConditionWaitOpened",
7252                json!({"sequence": 1, "condition_key": "signal:go"}),
7253            ),
7254            history_event(
7255                "SignalReceived",
7256                json!({
7257                    "signal_name": "go",
7258                    "arguments": ["now"],
7259                }),
7260            ),
7261            history_event(
7262                "TimerScheduled",
7263                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7264            ),
7265            history_event(
7266                "TimerFired",
7267                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7268            ),
7269        ];
7270
7271        let ctx = workflow_context(signal_then_timer.clone());
7272        let mut signal = Box::pin(ctx.wait_signal("go"));
7273        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7274        assert!(matches!(
7275            signal.as_mut().poll(&mut task_context),
7276            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
7277        ));
7278        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
7279        assert!(matches!(
7280            timer.as_mut().poll(&mut task_context),
7281            Poll::Ready(Ok(()))
7282        ));
7283        ctx.ensure_history_consumed()
7284            .expect("signal and timer history consumed in order");
7285
7286        let reordered = workflow_context(signal_then_timer);
7287        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
7288        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7289            timer_first.as_mut().poll(&mut task_context)
7290        else {
7291            panic!("timer cannot consume signal-wait-first history");
7292        };
7293        assert_eq!(failure.reason, "recorded_command_mismatch");
7294        assert_eq!(failure.sequence, Some(1));
7295        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
7296
7297        let timer_then_signal = vec![
7298            history_event(
7299                "TimerScheduled",
7300                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7301            ),
7302            history_event(
7303                "TimerFired",
7304                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7305            ),
7306            history_event(
7307                "ConditionWaitOpened",
7308                json!({"sequence": 2, "condition_key": "signal:go"}),
7309            ),
7310            history_event(
7311                "SignalReceived",
7312                json!({"signal_name": "go", "arguments": []}),
7313            ),
7314        ];
7315        let reordered = workflow_context(timer_then_signal);
7316        let mut signal_first = Box::pin(reordered.wait_signal("go"));
7317        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7318            signal_first.as_mut().poll(&mut task_context)
7319        else {
7320            panic!("signal wait cannot consume timer-first history");
7321        };
7322        assert_eq!(failure.reason, "recorded_command_mismatch");
7323        assert_eq!(failure.sequence, Some(1));
7324        assert_eq!(failure.expected.as_deref(), Some("timer"));
7325    }
7326
7327    #[test]
7328    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
7329        let duplicate_timer = WorkflowState::new(
7330            vec![
7331                history_event(
7332                    "TimerScheduled",
7333                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7334                ),
7335                history_event(
7336                    "TimerScheduled",
7337                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7338                ),
7339            ],
7340            "rust-workers".to_string(),
7341            JSON_CODEC.to_string(),
7342            None,
7343        )
7344        .expect_err("one workflow sequence cannot schedule two timers");
7345        assert!(matches!(
7346            duplicate_timer,
7347            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7348                if reason == "timer_schedule_missing_or_duplicate"
7349        ));
7350
7351        let colliding_kinds = WorkflowState::new(
7352            vec![
7353                history_event(
7354                    "TimerScheduled",
7355                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7356                ),
7357                history_event(
7358                    "ActivityCompleted",
7359                    json!({"sequence": 1, "activity_type": "same-sequence"}),
7360                ),
7361            ],
7362            "rust-workers".to_string(),
7363            JSON_CODEC.to_string(),
7364            None,
7365        )
7366        .expect_err("one workflow sequence cannot identify two command kinds");
7367        assert!(matches!(
7368            colliding_kinds,
7369            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7370                if reason == "durable_command_sequence_collision"
7371        ));
7372
7373        let duplicate_signal_wait = WorkflowState::new(
7374            vec![
7375                history_event(
7376                    "SignalWaitOpened",
7377                    json!({"sequence": 1, "signal_name": "go"}),
7378                ),
7379                history_event(
7380                    "SignalWaitOpened",
7381                    json!({"sequence": 1, "signal_name": "go"}),
7382                ),
7383            ],
7384            "rust-workers".to_string(),
7385            JSON_CODEC.to_string(),
7386            None,
7387        )
7388        .expect_err("one workflow sequence cannot open two signal waits");
7389        assert!(matches!(
7390            duplicate_signal_wait,
7391            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7392                if reason == "duplicate_signal_wait_open"
7393        ));
7394    }
7395
7396    #[test]
7397    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
7398        let ctx = workflow_context(Vec::new());
7399        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
7400        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7401        assert!(matches!(
7402            sleep.as_mut().poll(&mut task_context),
7403            Poll::Ready(Err(Error::TimerDurationOverflow))
7404        ));
7405        assert!(ctx.take_commands().expect("commands").is_empty());
7406    }
7407
7408    #[test]
7409    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
7410        let client = Client::new("http://127.0.0.1:8080").expect("client");
7411        let mut worker = Worker::new(client, "rust-workers");
7412        worker.register_workflow("rust.timer", |ctx, _input| async move {
7413            ctx.sleep(Duration::from_secs(5)).await?;
7414            ctx.activity("after-timer", json!([])).await
7415        });
7416
7417        let task = |history_events| WorkflowTask {
7418            task_id: "wft-rust-timer-1".to_string(),
7419            workflow_id: Some("wf-rust-timer".to_string()),
7420            run_id: Some("run-rust-timer".to_string()),
7421            workflow_type: "rust.timer".to_string(),
7422            payload_codec: JSON_CODEC.to_string(),
7423            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7424            history_events,
7425            total_history_events: None,
7426            next_history_page_token: None,
7427            workflow_task_attempt: 1,
7428            workflow_signal_id: None,
7429            signal_name: None,
7430            signal_arguments: None,
7431            lease_owner: Some("rust-worker".to_string()),
7432        };
7433
7434        let initial = worker
7435            .execute_workflow_task(task(Vec::new()))
7436            .expect("initial timer task");
7437        assert_eq!(
7438            initial,
7439            vec![json!({"type": "start_timer", "delay_seconds": 5})]
7440        );
7441
7442        let replayed = worker
7443            .execute_workflow_task(task(vec![
7444                history_event(
7445                    "TimerScheduled",
7446                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7447                ),
7448                history_event(
7449                    "TimerFired",
7450                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7451                ),
7452                history_event(
7453                    "ActivityCompleted",
7454                    json!({
7455                        "sequence": 2,
7456                        "activity_type": "after-timer",
7457                        "payload_codec": "json",
7458                        "result": {"codec": "json", "blob": "\"done\""},
7459                    }),
7460                ),
7461            ]))
7462            .expect("replayed workflow task");
7463        assert_eq!(replayed.len(), 1);
7464        assert_eq!(replayed[0]["type"], "complete_workflow");
7465        assert_eq!(
7466            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
7467            json!("done")
7468        );
7469    }
7470
7471    #[test]
7472    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
7473        let client = Client::new("http://127.0.0.1:8080").expect("client");
7474        let mut worker = Worker::new(client, "rust-workers");
7475        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
7476            ctx.sleep(Duration::from_secs(5)).await?;
7477            Ok(json!({"status": "timer fired"}))
7478        });
7479
7480        let task = WorkflowTask {
7481            task_id: "wft-rust-timer-pending".to_string(),
7482            workflow_id: Some("wf-rust-timer".to_string()),
7483            run_id: Some("run-rust-timer".to_string()),
7484            workflow_type: "rust.timer.pending".to_string(),
7485            payload_codec: JSON_CODEC.to_string(),
7486            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7487            history_events: vec![history_event(
7488                "TimerScheduled",
7489                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7490            )],
7491            total_history_events: Some(1),
7492            next_history_page_token: None,
7493            workflow_task_attempt: 1,
7494            workflow_signal_id: None,
7495            signal_name: None,
7496            signal_arguments: None,
7497            lease_owner: Some("rust-worker".to_string()),
7498        };
7499
7500        for _redelivery_or_restart in 0..2 {
7501            let commands = worker
7502                .execute_workflow_task(task.clone())
7503                .expect("recorded timer remains pending");
7504            assert!(
7505                commands.is_empty(),
7506                "recorded timer must not be rescheduled"
7507            );
7508        }
7509    }
7510
7511    #[test]
7512    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
7513        let client = Client::new("http://127.0.0.1:8080").expect("client");
7514        let mut worker = Worker::new(client, "rust-workers");
7515        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
7516            Ok(json!({"status": "completed"}))
7517        });
7518        let task = WorkflowTask {
7519            task_id: "wft-rust-timer-removed".to_string(),
7520            workflow_id: Some("wf-rust-timer".to_string()),
7521            run_id: Some("run-rust-timer".to_string()),
7522            workflow_type: "rust.timer.removed".to_string(),
7523            payload_codec: JSON_CODEC.to_string(),
7524            arguments: Some(json!({"codec": "json", "blob": "[]"})),
7525            history_events: vec![
7526                history_event(
7527                    "TimerScheduled",
7528                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7529                ),
7530                history_event(
7531                    "TimerFired",
7532                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7533                ),
7534            ],
7535            total_history_events: Some(2),
7536            next_history_page_token: None,
7537            workflow_task_attempt: 1,
7538            workflow_signal_id: None,
7539            signal_name: None,
7540            signal_arguments: None,
7541            lease_owner: Some("rust-worker".to_string()),
7542        };
7543
7544        let Error::NonDeterministicReplay(failure) = worker
7545            .execute_workflow_task(task)
7546            .expect_err("removed timer must fail replay")
7547        else {
7548            panic!("expected typed replay failure");
7549        };
7550        assert_eq!(failure.reason, "recorded_commands_unconsumed");
7551        assert_eq!(failure.sequence, Some(1));
7552    }
7553
7554    #[test]
7555    fn workflow_context_emits_explicit_child_workflow_contract() {
7556        let ctx = WorkflowContext {
7557            state: Arc::new(Mutex::new(
7558                WorkflowState::new_with_identity(
7559                    Vec::new(),
7560                    Some("wf-parent".to_string()),
7561                    Some("run-parent".to_string()),
7562                    "parent-workers".to_string(),
7563                    JSON_CODEC.to_string(),
7564                    None,
7565                )
7566                .expect("workflow state"),
7567            )),
7568        };
7569        let options = ChildWorkflowOptions::new("python-workers")
7570            .parent_close_policy(ParentClosePolicy::RequestCancel)
7571            .retry_policy(ChildWorkflowRetryPolicy {
7572                max_attempts: Some(3),
7573                backoff_seconds: vec![1, 5],
7574                non_retryable_error_types: vec!["ValidationError".to_string()],
7575            })
7576            .execution_timeout_seconds(600)
7577            .run_timeout_seconds(120);
7578        let mut call = Box::pin(ctx.start_child_workflow(
7579            "python.fulfil-order",
7580            options,
7581            json!([{"order_id": "order-42"}]),
7582        ));
7583        let mut task_context = TaskContext::from_waker(noop_waker_ref());
7584
7585        assert!(matches!(
7586            call.as_mut().poll(&mut task_context),
7587            Poll::Pending
7588        ));
7589        let commands = ctx.take_commands().expect("commands");
7590        assert_eq!(commands.len(), 1);
7591        let command = &commands[0];
7592        assert_eq!(command["type"], "start_child_workflow");
7593        assert_eq!(command["workflow_type"], "python.fulfil-order");
7594        assert_eq!(command["queue"], "python-workers");
7595        assert_eq!(command["parent_close_policy"], "request_cancel");
7596        assert_eq!(command["retry_policy"]["max_attempts"], 3);
7597        assert_eq!(command["execution_timeout_seconds"], 600);
7598        assert_eq!(command["run_timeout_seconds"], 120);
7599        assert_eq!(
7600            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
7601            json!([{"order_id": "order-42"}])
7602        );
7603    }
7604
7605    fn child_parent_worker() -> Worker {
7606        let client = Client::new("http://127.0.0.1:8080").expect("client");
7607        let mut worker = Worker::new(client, "rust-parent-workers");
7608        worker.register_workflow("rust.parent", |ctx, _input| async move {
7609            let child = ctx
7610                .start_child_workflow(
7611                    "python.child",
7612                    ChildWorkflowOptions::new("python-child-workers")
7613                        .parent_close_policy(ParentClosePolicy::Terminate),
7614                    json!([{"codec_probe": [1, true, "rust"]}]),
7615                )
7616                .await?;
7617            Ok(json!({
7618                "parent_workflow_id": child.parent.workflow_id,
7619                "parent_run_id": child.parent.run_id,
7620                "child_workflow_id": child.child.workflow_id,
7621                "child_run_id": child.child.run_id,
7622                "child_workflow_type": child.child_workflow_type,
7623                "result": child.result,
7624            }))
7625        });
7626        worker
7627    }
7628
7629    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
7630        WorkflowTask {
7631            task_id: "wft-child-parent".to_string(),
7632            workflow_id: Some("wf-parent".to_string()),
7633            run_id: Some("run-parent".to_string()),
7634            workflow_type: "rust.parent".to_string(),
7635            payload_codec: JSON_CODEC.to_string(),
7636            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
7637            history_events: vec![
7638                HistoryEvent {
7639                    event_type: "ChildWorkflowScheduled".to_string(),
7640                    payload: json!({
7641                        "sequence": 1,
7642                        "child_call_id": "call-child",
7643                        "child_workflow_instance_id": "wf-child",
7644                        "child_workflow_run_id": "run-child",
7645                        "child_workflow_type": "python.child",
7646                    }),
7647                    raw: HashMap::new(),
7648                },
7649                HistoryEvent {
7650                    event_type: event_type.to_string(),
7651                    payload,
7652                    raw: HashMap::new(),
7653                },
7654            ],
7655            total_history_events: Some(2),
7656            next_history_page_token: None,
7657            workflow_task_attempt: 1,
7658            workflow_signal_id: None,
7659            signal_name: None,
7660            signal_arguments: None,
7661            lease_owner: Some("rust-worker".to_string()),
7662        }
7663    }
7664
7665    #[test]
7666    fn committed_child_result_replays_without_starting_a_duplicate() {
7667        let worker = child_parent_worker();
7668        let task = child_parent_task(
7669            "ChildRunCompleted",
7670            json!({
7671                "sequence": 1,
7672                "child_call_id": "call-child",
7673                "child_workflow_instance_id": "wf-child",
7674                "child_workflow_run_id": "run-child",
7675                "child_workflow_type": "python.child",
7676                "payload_codec": "json",
7677                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
7678            }),
7679        );
7680
7681        for _restart in 0..2 {
7682            let commands = worker
7683                .execute_workflow_task(task.clone())
7684                .expect("replayed parent task");
7685            assert_eq!(commands.len(), 1);
7686            assert_eq!(commands[0]["type"], "complete_workflow");
7687            assert!(!commands
7688                .iter()
7689                .any(|command| command["type"] == "start_child_workflow"));
7690            let output =
7691                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7692            assert_eq!(output["parent_workflow_id"], "wf-parent");
7693            assert_eq!(output["parent_run_id"], "run-parent");
7694            assert_eq!(output["child_workflow_id"], "wf-child");
7695            assert_eq!(output["child_run_id"], "run-child");
7696            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
7697        }
7698    }
7699
7700    #[test]
7701    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
7702        let worker = child_parent_worker();
7703        let mut task = child_parent_task("unused", Value::Null);
7704        task.history_events.truncate(1);
7705        task.total_history_events = Some(1);
7706
7707        for _redelivery_or_restart in 0..2 {
7708            let commands = worker
7709                .execute_workflow_task(task.clone())
7710                .expect("recorded child remains pending");
7711            assert!(
7712                commands.is_empty(),
7713                "recorded pending child must not be started again"
7714            );
7715        }
7716    }
7717
7718    #[test]
7719    fn child_cancellation_becomes_stable_parent_failure_command() {
7720        let worker = child_parent_worker();
7721        let task = child_parent_task(
7722            "ChildRunCancelled",
7723            json!({
7724                "sequence": 1,
7725                "child_workflow_instance_id": "wf-child",
7726                "child_workflow_run_id": "run-child",
7727                "child_workflow_type": "python.child",
7728                "failure_id": "failure-child",
7729                "failure_category": "cancelled",
7730                "message": "cancelled by parent-close policy",
7731            }),
7732        );
7733
7734        let commands = worker
7735            .execute_workflow_task(task)
7736            .expect("parent settlement");
7737        assert_eq!(commands.len(), 1);
7738        assert_eq!(commands[0]["type"], "fail_workflow");
7739        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
7740        assert_eq!(
7741            commands[0]["exception"]["properties"]["reason"],
7742            "cancelled"
7743        );
7744        assert_eq!(
7745            commands[0]["exception"]["properties"]["child_workflow_run_id"],
7746            "run-child"
7747        );
7748    }
7749
7750    #[test]
7751    fn workflow_can_handle_typed_child_failure() {
7752        let client = Client::new("http://127.0.0.1:8080").expect("client");
7753        let mut worker = Worker::new(client, "rust-parent-workers");
7754        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
7755            match ctx
7756                .start_child_workflow(
7757                    "python.child",
7758                    ChildWorkflowOptions::new("python-child-workers"),
7759                    json!([]),
7760                )
7761                .await
7762            {
7763                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
7764                    "reason": failure.reason,
7765                    "failure_id": failure.failure_id,
7766                    "exception_class": failure.exception_class,
7767                    "child_run_id": failure.child_workflow_run_id,
7768                })),
7769                Err(error) => Err(error),
7770                Ok(_) => Err(Error::WorkerLoop(
7771                    "child unexpectedly succeeded".to_string(),
7772                )),
7773            }
7774        });
7775        let mut task = child_parent_task(
7776            "ChildRunFailed",
7777            json!({
7778                "sequence": 1,
7779                "child_workflow_instance_id": "wf-child",
7780                "child_workflow_run_id": "run-child",
7781                "child_workflow_type": "python.child",
7782                "failure_id": "failure-child",
7783                "failure_category": "child_workflow",
7784                "message": "payment rejected",
7785                "exception": {
7786                    "type": "PaymentRejected",
7787                    "class": "payments.PaymentRejected",
7788                    "message": "payment rejected"
7789                }
7790            }),
7791        );
7792        task.workflow_type = "rust.handled-parent".to_string();
7793
7794        let commands = worker.execute_workflow_task(task).expect("handled failure");
7795        assert_eq!(commands[0]["type"], "complete_workflow");
7796        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7797        assert_eq!(output["reason"], "child_workflow");
7798        assert_eq!(output["failure_id"], "failure-child");
7799        assert_eq!(output["exception_class"], "payments.PaymentRejected");
7800        assert_eq!(output["child_run_id"], "run-child");
7801    }
7802
7803    #[test]
7804    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
7805        let client = Client::new("http://127.0.0.1:8080").expect("client");
7806        let mut worker = Worker::new(client, "rust-workers");
7807
7808        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
7809            let signal = ctx.wait_signal("start").await?;
7810            let name = signal
7811                .first()
7812                .and_then(|value| value.as_str())
7813                .unwrap_or("world");
7814            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
7815            Ok(json!({
7816                "greeting": greeting,
7817                "language": "rust"
7818            }))
7819        });
7820
7821        let signal_arguments =
7822            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
7823        let task = WorkflowTask {
7824            task_id: "wft-rust-signal-1".to_string(),
7825            workflow_id: Some("wf-rust-hello".to_string()),
7826            run_id: Some("run-rust-hello".to_string()),
7827            workflow_type: "rust.hello_workflow".to_string(),
7828            payload_codec: DEFAULT_CODEC.to_string(),
7829            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7830            history_events: vec![HistoryEvent {
7831                event_type: "SignalReceived".to_string(),
7832                payload: json!({
7833                    "signal_id": "sig-rust-1",
7834                    "signal_name": "start"
7835                }),
7836                raw: HashMap::new(),
7837            }],
7838            total_history_events: Some(1),
7839            next_history_page_token: None,
7840            workflow_task_attempt: 1,
7841            workflow_signal_id: Some("sig-rust-1".to_string()),
7842            signal_name: Some("start".to_string()),
7843            signal_arguments: Some(signal_arguments),
7844            lease_owner: Some("rust-worker".to_string()),
7845        };
7846
7847        let commands = worker.execute_workflow_task(task).expect("workflow task");
7848
7849        assert_eq!(commands.len(), 1);
7850        assert_eq!(commands[0]["type"], "schedule_activity");
7851        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
7852        assert_eq!(
7853            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
7854            json!(["Rust"])
7855        );
7856    }
7857
7858    #[test]
7859    fn workflow_task_appends_paginated_history_events() {
7860        let mut task = WorkflowTask {
7861            task_id: "wft-rust-pages-1".to_string(),
7862            workflow_id: Some("wf-rust-pages".to_string()),
7863            run_id: Some("run-rust-pages".to_string()),
7864            workflow_type: "rust.hello_workflow".to_string(),
7865            payload_codec: DEFAULT_CODEC.to_string(),
7866            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7867            history_events: vec![HistoryEvent {
7868                event_type: "WorkflowStarted".to_string(),
7869                payload: json!({}),
7870                raw: HashMap::new(),
7871            }],
7872            total_history_events: Some(3),
7873            next_history_page_token: Some("MQ==".to_string()),
7874            workflow_task_attempt: 1,
7875            workflow_signal_id: None,
7876            signal_name: None,
7877            signal_arguments: None,
7878            lease_owner: Some("rust-worker".to_string()),
7879        };
7880
7881        task.append_history_page(WorkflowTaskHistoryPage {
7882            history_events: vec![
7883                HistoryEvent {
7884                    event_type: "SignalReceived".to_string(),
7885                    payload: json!({
7886                        "signal_id": "sig-rust-1",
7887                        "signal_name": "start",
7888                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
7889                            .expect("signal arguments")
7890                    }),
7891                    raw: HashMap::new(),
7892                },
7893                HistoryEvent {
7894                    event_type: "MarkerRecorded".to_string(),
7895                    payload: json!({"sequence": 3}),
7896                    raw: HashMap::new(),
7897                },
7898            ],
7899            total_history_events: Some(3),
7900            next_history_page_token: None,
7901        });
7902
7903        assert_eq!(task.history_events.len(), 3);
7904        assert_eq!(task.total_history_events, Some(3));
7905        assert_eq!(task.next_history_page_token, None);
7906
7907        let signals =
7908            signal_values(&task.history_events, "start", DEFAULT_CODEC, None).expect("signals");
7909        assert_eq!(signals, vec![vec![json!("Rust")]]);
7910    }
7911
7912    #[tokio::test]
7913    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
7914        let client = Client::new("http://127.0.0.1:8080").expect("client");
7915        let mut worker = Worker::new(client, "rust-workers");
7916        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
7917        worker.register_query("counter", "current", |ctx, _args| async move {
7918            let mut count = 0_i64;
7919            for signal in ctx.signal_events() {
7920                let value = signal
7921                    .arguments
7922                    .first()
7923                    .and_then(Value::as_i64)
7924                    .unwrap_or_default();
7925                match signal.name.as_str() {
7926                    "increment" => count += value,
7927                    "set" => count = value,
7928                    _ => {}
7929                }
7930            }
7931            Ok(json!(count))
7932        });
7933
7934        let task = QueryTask {
7935            query_task_id: "query-rust-counter".to_string(),
7936            query_task_attempt: 1,
7937            lease_owner: Some("rust-worker".to_string()),
7938            workflow_id: Some("counter-1".to_string()),
7939            run_id: Some("run-counter-1".to_string()),
7940            workflow_type: "counter".to_string(),
7941            query_name: "current".to_string(),
7942            payload_codec: DEFAULT_CODEC.to_string(),
7943            workflow_arguments: Some(
7944                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
7945            ),
7946            query_arguments: Some(
7947                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
7948            ),
7949            history_events: vec![
7950                HistoryEvent {
7951                    event_type: "SignalReceived".to_string(),
7952                    payload: json!({
7953                        "signal_id": "php-signal-1",
7954                        "signal_name": "increment",
7955                        "workflow_sequence": 1,
7956                        "payload_codec": DEFAULT_CODEC,
7957                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
7958                    }),
7959                    raw: HashMap::new(),
7960                },
7961                HistoryEvent {
7962                    event_type: "SignalReceived".to_string(),
7963                    payload: json!({
7964                        "signal_id": "python-signal-2",
7965                        "signal_name": "increment",
7966                        "workflow_sequence": 2,
7967                        "payload_codec": JSON_CODEC,
7968                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
7969                    }),
7970                    raw: HashMap::new(),
7971                },
7972                HistoryEvent {
7973                    event_type: "SignalReceived".to_string(),
7974                    payload: json!({
7975                        "signal_id": "rust-signal-3",
7976                        "signal_name": "set",
7977                        "workflow_sequence": 3,
7978                        "payload_codec": DEFAULT_CODEC,
7979                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
7980                    }),
7981                    raw: HashMap::new(),
7982                },
7983            ],
7984            history_export: None,
7985            run_status: Some("completed".to_string()),
7986        };
7987
7988        let result = worker.execute_query_task(task).await.expect("query result");
7989        assert_eq!(result, json!(0));
7990    }
7991
7992    #[tokio::test]
7993    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
7994        let worker = replay_counter_worker();
7995        let running_history = json!([
7996            {
7997                "type": "ActivityCompleted",
7998                "payload": {
7999                    "sequence": 1,
8000                    "activity_type": "load-counter",
8001                    "payload_codec": "json",
8002                    "result": {"codec": "json", "blob": "\"loaded\""}
8003                }
8004            },
8005            {
8006                "type": "SignalReceived",
8007                "payload": {
8008                    "signal_id": "signal-3",
8009                    "signal_name": "increment",
8010                    "payload_codec": "json",
8011                    "arguments": {"codec": "json", "blob": "[3]"}
8012                }
8013            }
8014        ]);
8015
8016        let running = worker
8017            .execute_query_task(replay_counter_query(
8018                "current",
8019                running_history.clone(),
8020                "running",
8021            ))
8022            .await
8023            .expect("running replay query");
8024        assert_eq!(
8025            running,
8026            json!({"loaded": "loaded", "count": 3, "finished": false})
8027        );
8028
8029        let detached = worker
8030            .execute_query_task(replay_counter_query(
8031                "detached-mutation",
8032                running_history.clone(),
8033                "running",
8034            ))
8035            .await
8036            .expect("query mutates only its detached state clone");
8037        assert_eq!(detached, json!(999));
8038        let failed = worker
8039            .execute_query_task(replay_counter_query(
8040                "failed-mutation",
8041                running_history.clone(),
8042                "running",
8043            ))
8044            .await
8045            .expect_err("failed query");
8046        assert_eq!(failed.reason, "query_rejected");
8047        let unchanged = worker
8048            .execute_query_task(replay_counter_query("current", running_history, "running"))
8049            .await
8050            .expect("later query reconstructs unchanged state");
8051        assert_eq!(unchanged, running);
8052
8053        let restarted_worker = replay_counter_worker();
8054        let restarted_task: QueryTask = serde_json::from_value(json!({
8055            "query_task_id": "query-after-restart",
8056            "workflow_id": "counter-1",
8057            "run_id": "run-counter-1",
8058            "workflow_type": "replay-counter",
8059            "query_name": "current",
8060            "payload_codec": "json",
8061            "workflow_arguments": {"codec": "json", "blob": "[]"},
8062            "query_arguments": {"codec": "json", "blob": "[]"},
8063            "history_events": [],
8064            "history_export": {
8065                "payloads": {"codec": "json"},
8066                "history_events": [
8067                    {
8068                        "type": "ActivityCompleted",
8069                        "payload": {
8070                            "sequence": 1,
8071                            "activity_type": "load-counter",
8072                            "payload_codec": "json",
8073                            "result": null
8074                        }
8075                    },
8076                    {
8077                        "type": "SignalReceived",
8078                        "payload": {"signal_id": "signal-3", "signal_name": "increment"}
8079                    },
8080                    {
8081                        "type": "SignalReceived",
8082                        "payload": {"signal_id": "signal-5", "signal_name": "increment"}
8083                    }
8084                ],
8085                "activities": [{
8086                    "sequence": 1,
8087                    "activity_type": "load-counter",
8088                    "payload_codec": "json",
8089                    "result": {"codec": "json", "blob": "\"loaded\""}
8090                }],
8091                "signals": [
8092                    {
8093                        "id": "signal-3",
8094                        "name": "increment",
8095                        "payload_codec": "json",
8096                        "arguments": "[3]"
8097                    },
8098                    {
8099                        "id": "signal-5",
8100                        "name": "increment",
8101                        "payload_codec": "json",
8102                        "arguments": "[5]"
8103                    }
8104                ]
8105            },
8106            "run_status": "completed"
8107        }))
8108        .expect("cold replay query task");
8109        let completed = restarted_worker
8110            .execute_query_task(restarted_task)
8111            .await
8112            .expect("completed cold replay query");
8113        assert_eq!(
8114            completed,
8115            json!({"loaded": "loaded", "count": 8, "finished": true})
8116        );
8117    }
8118
8119    #[tokio::test]
8120    async fn replayed_query_replay_failures_are_machine_readable() {
8121        let worker = replay_counter_worker();
8122        let task = replay_counter_query(
8123            "current",
8124            json!([{
8125                "type": "ActivityCompleted",
8126                "payload": {
8127                    "sequence": 1,
8128                    "payload_codec": "json",
8129                    "result": {"codec": "json", "blob": "{"}
8130                }
8131            }]),
8132            "running",
8133        );
8134        let failure = worker
8135            .execute_query_task(task)
8136            .await
8137            .expect_err("invalid replay history payload");
8138        assert_eq!(failure.reason, "query_workflow_state_unavailable");
8139        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
8140    }
8141
8142    #[tokio::test]
8143    async fn query_task_restores_compact_history_from_export() {
8144        let client = Client::new("http://127.0.0.1:8080").expect("client");
8145        let mut worker = Worker::new(client, "rust-workers");
8146        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8147        worker.register_query("counter", "current", |ctx, _args| async move {
8148            Ok(json!(ctx.signals("increment")[0][0]))
8149        });
8150        let task: QueryTask = serde_json::from_value(json!({
8151            "query_task_id": "query-export",
8152            "workflow_type": "counter",
8153            "query_name": "current",
8154            "payload_codec": "json",
8155            "workflow_arguments": {"codec": "json", "blob": "[]"},
8156            "query_arguments": {"codec": "json", "blob": "[]"},
8157            "history_events": [],
8158            "history_export": {
8159                "payloads": {"codec": "json"},
8160                "history_events": [{
8161                    "type": "SignalReceived",
8162                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
8163                }],
8164                "signals": [{
8165                    "id": "signal-export",
8166                    "name": "increment",
8167                    "status": "applied",
8168                    "workflow_sequence": 1,
8169                    "payload_codec": "json",
8170                    "arguments": "[9]"
8171                }]
8172            }
8173        }))
8174        .expect("query task");
8175
8176        let result = worker.execute_query_task(task).await.expect("query result");
8177        assert_eq!(result, json!(9));
8178    }
8179
8180    #[tokio::test]
8181    async fn query_task_failures_have_stable_reasons() {
8182        let client = Client::new("http://127.0.0.1:8080").expect("client");
8183        let mut worker = Worker::new(client, "rust-workers");
8184        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8185        worker.register_query(
8186            "counter",
8187            "current",
8188            |_ctx, _args| async move { Ok(json!(0)) },
8189        );
8190
8191        let base_task = QueryTask {
8192            query_task_id: "query-errors".to_string(),
8193            query_task_attempt: 1,
8194            lease_owner: None,
8195            workflow_id: Some("counter-errors".to_string()),
8196            run_id: Some("run-errors".to_string()),
8197            workflow_type: "counter".to_string(),
8198            query_name: "missing".to_string(),
8199            payload_codec: JSON_CODEC.to_string(),
8200            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8201            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8202            history_events: Vec::new(),
8203            history_export: None,
8204            run_status: Some("running".to_string()),
8205        };
8206
8207        let unknown = worker
8208            .execute_query_task(base_task.clone())
8209            .await
8210            .expect_err("unknown query");
8211        assert_eq!(unknown.reason, "rejected_unknown_query");
8212
8213        let mut malformed = base_task;
8214        malformed.query_name = "current".to_string();
8215        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
8216        let malformed = worker
8217            .execute_query_task(malformed)
8218            .await
8219            .expect_err("malformed payload");
8220        assert_eq!(malformed.reason, "query_payload_decode_failed");
8221
8222        let client = Client::new("http://127.0.0.1:8080").expect("client");
8223        let mut unavailable_worker = Worker::new(client, "rust-workers");
8224        unavailable_worker
8225            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8226        let unavailable_task: QueryTask = serde_json::from_value(json!({
8227            "query_task_id": "query-unavailable",
8228            "workflow_type": "counter",
8229            "query_name": "current",
8230            "payload_codec": "json",
8231            "workflow_arguments": {"codec": "json", "blob": "[]"},
8232            "query_arguments": {"codec": "json", "blob": "[]"}
8233        }))
8234        .expect("query task");
8235        let unavailable = unavailable_worker
8236            .execute_query_task(unavailable_task)
8237            .await
8238            .expect_err("query handler unavailable");
8239        assert_eq!(unavailable.reason, "query_handler_unavailable");
8240    }
8241
8242    #[tokio::test]
8243    async fn client_query_decodes_result_and_typed_failure() {
8244        let server = MockWorkerServer::start();
8245        let client = Client::builder(server.base_url())
8246            .timeout(Duration::from_secs(2))
8247            .build()
8248            .expect("client");
8249
8250        let result = client
8251            .query_workflow("counter-1", "current", json!([]))
8252            .await
8253            .expect("query result");
8254        assert_eq!(result, json!({"count": 8}));
8255
8256        let error = client
8257            .query_workflow("counter-1", "missing", json!([]))
8258            .await
8259            .expect_err("unknown query");
8260        let Error::QueryFailed(failure) = error else {
8261            panic!("expected typed query failure");
8262        };
8263        assert_eq!(failure.status, 404);
8264        assert_eq!(failure.reason, "rejected_unknown_query");
8265    }
8266
8267    #[tokio::test]
8268    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
8269        let server = MockWorkerServer::start();
8270        let client = Client::builder(server.base_url())
8271            .timeout(Duration::from_secs(2))
8272            .build()
8273            .expect("client");
8274
8275        let options = WorkflowCommandOptions::new()
8276            .reason("cleanup requested")
8277            .request_id("cancel-17");
8278        let cancelled = client
8279            .cancel_workflow("wf-lifecycle", options)
8280            .await
8281            .expect("instance cancellation");
8282        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
8283        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
8284        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
8285        assert_eq!(
8286            server.request_body("/api/workflows/wf-lifecycle/cancel"),
8287            json!({"reason":"cleanup requested","request_id":"cancel-17"})
8288        );
8289
8290        let terminated = client
8291            .terminate_workflow(
8292                "wf-lifecycle",
8293                WorkflowCommandOptions::new().reason("forced stop"),
8294            )
8295            .await
8296            .expect("instance termination");
8297        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
8298        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
8299
8300        client
8301            .cancel_workflow_run(
8302                "wf-lifecycle",
8303                "run-current",
8304                WorkflowCommandOptions::default(),
8305            )
8306            .await
8307            .expect("selected run cancellation");
8308        client
8309            .terminate_workflow_run(
8310                "wf-lifecycle",
8311                "run-current",
8312                WorkflowCommandOptions::default(),
8313            )
8314            .await
8315            .expect("selected run termination");
8316
8317        for (command, error) in [
8318            (
8319                WorkflowCommandKind::Cancel,
8320                client
8321                    .cancel_workflow_run(
8322                        "wf-lifecycle",
8323                        "run-stale",
8324                        WorkflowCommandOptions::default(),
8325                    )
8326                    .await
8327                    .expect_err("stale cancellation must be rejected"),
8328            ),
8329            (
8330                WorkflowCommandKind::Terminate,
8331                client
8332                    .terminate_workflow_run(
8333                        "wf-lifecycle",
8334                        "run-stale",
8335                        WorkflowCommandOptions::default(),
8336                    )
8337                    .await
8338                    .expect_err("stale termination must be rejected"),
8339            ),
8340        ] {
8341            let Error::WorkflowCommandRejected(rejection) = error else {
8342                panic!("expected typed command rejection");
8343            };
8344            assert_eq!(rejection.command, command);
8345            assert_eq!(rejection.status, 409);
8346            assert_eq!(rejection.reason, "historical_run_command_rejected");
8347            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
8348            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
8349        }
8350    }
8351
8352    #[tokio::test]
8353    async fn workflow_start_options_send_server_enforced_deadlines() {
8354        let server = MockWorkerServer::start();
8355        let client = Client::builder(server.base_url())
8356            .timeout(Duration::from_secs(2))
8357            .build()
8358            .expect("client");
8359
8360        let handle = client
8361            .start_workflow_with_options(
8362                "rust.timeout",
8363                "rust-timeouts",
8364                "wf-start-options",
8365                WorkflowStartOptions::new()
8366                    .execution_timeout_seconds(30)
8367                    .run_timeout_seconds(1),
8368                json!([]),
8369            )
8370            .await
8371            .expect("workflow start");
8372
8373        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
8374        let body = server.request_body("/api/workflows");
8375        assert_eq!(body["execution_timeout_seconds"], 30);
8376        assert_eq!(body["run_timeout_seconds"], 1);
8377
8378        let invalid = client
8379            .start_workflow_with_options(
8380                "rust.timeout",
8381                "rust-timeouts",
8382                "wf-invalid-options",
8383                WorkflowStartOptions::new()
8384                    .execution_timeout_seconds(1)
8385                    .run_timeout_seconds(2),
8386                json!([]),
8387            )
8388            .await
8389            .expect_err("invalid deadline ordering");
8390        assert!(invalid
8391            .to_string()
8392            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
8393    }
8394
8395    #[tokio::test]
8396    async fn workflow_result_returns_each_typed_terminal_outcome() {
8397        let server = MockWorkerServer::start();
8398        let client = Client::builder(server.base_url())
8399            .timeout(Duration::from_secs(2))
8400            .build()
8401            .expect("client");
8402        let options = WorkflowResultOptions {
8403            poll_interval: Duration::ZERO,
8404            timeout: Duration::from_secs(1),
8405        };
8406
8407        let failed = WorkflowHandle {
8408            client: client.clone(),
8409            workflow_id: "wf-failed".to_string(),
8410            run_id: Some("run-failed".to_string()),
8411            workflow_type: "failure".to_string(),
8412        }
8413        .result(options)
8414        .await
8415        .expect_err("failed outcome");
8416        let Error::WorkflowFailed(failure) = failed else {
8417            panic!("expected WorkflowFailed");
8418        };
8419        assert_eq!(failure.workflow_id, "wf-failed");
8420        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
8421        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
8422        assert_eq!(failure.failure_category.as_deref(), Some("application"));
8423        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
8424        assert_eq!(
8425            failure.exception_class.as_deref(),
8426            Some("billing::PaymentError")
8427        );
8428        assert_eq!(failure.non_retryable, Some(true));
8429
8430        for (workflow_id, expected_kind, expected_reason) in [
8431            (
8432                "wf-cancelled",
8433                WorkflowTerminalKind::Cancelled,
8434                "cleanup requested",
8435            ),
8436            (
8437                "wf-terminated",
8438                WorkflowTerminalKind::Terminated,
8439                "forced stop",
8440            ),
8441            (
8442                "wf-timed-out",
8443                WorkflowTerminalKind::TimedOut,
8444                "run_timeout",
8445            ),
8446        ] {
8447            let error = WorkflowHandle {
8448                client: client.clone(),
8449                workflow_id: workflow_id.to_string(),
8450                run_id: None,
8451                workflow_type: "terminal".to_string(),
8452            }
8453            .result(options)
8454            .await
8455            .expect_err("typed terminal outcome");
8456            let outcome = match error {
8457                Error::WorkflowCancelled(outcome) => outcome,
8458                Error::WorkflowTerminated(outcome) => outcome,
8459                Error::WorkflowTimedOut(outcome) => outcome,
8460                other => panic!("unexpected terminal error: {other}"),
8461            };
8462            assert_eq!(outcome.kind, expected_kind);
8463            assert_eq!(outcome.workflow_id, workflow_id);
8464            assert_eq!(outcome.reason, expected_reason);
8465        }
8466
8467        let wait_timeout = WorkflowHandle {
8468            client,
8469            workflow_id: "wf-waiting".to_string(),
8470            run_id: Some("run-waiting".to_string()),
8471            workflow_type: "waiting".to_string(),
8472        }
8473        .result(WorkflowResultOptions {
8474            poll_interval: Duration::ZERO,
8475            timeout: Duration::ZERO,
8476        })
8477        .await
8478        .expect_err("client wait timeout");
8479        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
8480            panic!("expected typed client timeout");
8481        };
8482        assert_eq!(timeout.reason, "result_wait_timeout");
8483        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
8484        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
8485    }
8486
8487    #[tokio::test]
8488    async fn workflow_result_awaits_the_handle_selected_run() {
8489        let server = MockWorkerServer::start();
8490        let client = Client::builder(server.base_url())
8491            .timeout(Duration::from_secs(2))
8492            .build()
8493            .expect("client");
8494
8495        let error = WorkflowHandle {
8496            client,
8497            workflow_id: "wf-selected".to_string(),
8498            run_id: Some("run-selected".to_string()),
8499            workflow_type: "selected".to_string(),
8500        }
8501        .result(WorkflowResultOptions {
8502            poll_interval: Duration::ZERO,
8503            timeout: Duration::from_secs(1),
8504        })
8505        .await
8506        .expect_err("the selected run is cancelled even though the current run completed");
8507
8508        let Error::WorkflowCancelled(outcome) = error else {
8509            panic!("expected selected run cancellation");
8510        };
8511        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
8512        assert_eq!(outcome.reason, "selected run cancelled");
8513        assert_eq!(
8514            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
8515            1
8516        );
8517        assert_eq!(server.request_count("/api/workflows/wf-selected"), 0);
8518    }
8519
8520    #[tokio::test]
8521    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
8522        let server = MockWorkerServer::draining_polls();
8523        let client = Client::builder(server.base_url())
8524            .timeout(Duration::from_secs(2))
8525            .build()
8526            .expect("client");
8527
8528        let workflow = client
8529            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
8530            .await
8531            .expect("workflow drain response");
8532        let activity = client
8533            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
8534            .await
8535            .expect("activity drain response");
8536        let query = client
8537            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
8538            .await
8539            .expect("query drain response");
8540
8541        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
8542            assert_eq!(
8543                outcome,
8544                WorkerPollOutcome::Stop {
8545                    poll_status: Some("draining".to_string()),
8546                    reason: Some("worker_draining".to_string()),
8547                }
8548            );
8549        }
8550
8551        assert!(client
8552            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
8553            .await
8554            .expect("compatibility poll")
8555            .is_none());
8556    }
8557
8558    #[tokio::test]
8559    async fn managed_worker_honors_drain_stop_for_every_task_family() {
8560        let server = MockWorkerServer::draining_polls();
8561        let client = Client::builder(server.base_url())
8562            .timeout(Duration::from_secs(2))
8563            .build()
8564            .expect("client");
8565
8566        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
8567            .worker_id("draining-workflow-worker")
8568            .poll_timeout(Duration::ZERO);
8569        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
8570        workflow_worker
8571            .run()
8572            .await
8573            .expect("workflow drain is a clean stop");
8574
8575        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
8576            .worker_id("draining-activity-worker")
8577            .poll_timeout(Duration::ZERO);
8578        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
8579        activity_worker
8580            .run()
8581            .await
8582            .expect("activity drain is a clean stop");
8583
8584        let mut query_worker = Worker::new(client, "rust-workers")
8585            .worker_id("draining-query-worker")
8586            .poll_timeout(Duration::ZERO);
8587        query_worker.register_query("counter", "current", |_ctx, _args| async {
8588            Ok(Value::Null)
8589        });
8590        query_worker
8591            .run()
8592            .await
8593            .expect("query drain is a clean stop");
8594    }
8595
8596    #[tokio::test]
8597    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
8598        let server = MockWorkerServer::start();
8599        let client = Client::builder(server.base_url())
8600            .timeout(Duration::from_secs(2))
8601            .build()
8602            .expect("client");
8603
8604        let heartbeat = client
8605            .heartbeat_activity_task(
8606                "activity-cancel",
8607                "attempt-cancel",
8608                "rust-worker",
8609                json!({"stage":"cleanup"}),
8610            )
8611            .await
8612            .expect("cancellation heartbeat");
8613        assert!(heartbeat.cancel_requested);
8614        assert!(heartbeat.should_stop());
8615        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
8616        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
8617
8618        let error = client
8619            .complete_activity_task(
8620                "activity-cancel",
8621                "attempt-cancel",
8622                "rust-worker",
8623                json!({"late":true}),
8624                JSON_CODEC,
8625            )
8626            .await
8627            .expect_err("late completion must be refused");
8628        assert!(activity_task_rejection_is_final(&error));
8629        let Error::ActivityTaskRejected(rejection) = error else {
8630            panic!("expected typed activity rejection");
8631        };
8632        assert_eq!(rejection.status, 409);
8633        assert_eq!(rejection.reason, "run_cancelled");
8634        assert!(rejection.cancel_requested);
8635        assert_eq!(rejection.can_continue, Some(false));
8636    }
8637
8638    #[tokio::test]
8639    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
8640        let server = MockWorkerServer::cancelled_activity();
8641        let client = Client::builder(server.base_url())
8642            .timeout(Duration::from_secs(2))
8643            .build()
8644            .expect("client");
8645        let cancellation_observed = Arc::new(AtomicBool::new(false));
8646        let observed = Arc::clone(&cancellation_observed);
8647        let mut worker = Worker::new(client.clone(), "rust-workers")
8648            .worker_id("rust-cancel-worker")
8649            .poll_timeout(Duration::from_millis(10));
8650        worker.register_activity("cancel-aware", move |ctx, _args| {
8651            let observed = Arc::clone(&observed);
8652            async move {
8653                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
8654                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
8655                Ok(json!({"late":"completion"}))
8656            }
8657        });
8658
8659        assert_eq!(
8660            worker.run_once().await.expect("cancelled attempt handled"),
8661            1
8662        );
8663        assert!(cancellation_observed.load(Ordering::SeqCst));
8664        assert_eq!(
8665            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
8666            1
8667        );
8668
8669        let mut restarted = Worker::new(client, "rust-workers")
8670            .worker_id("rust-cancel-worker-restarted")
8671            .poll_timeout(Duration::from_millis(10));
8672        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
8673        assert_eq!(
8674            restarted
8675                .run_once()
8676                .await
8677                .expect("replacement worker continues polling"),
8678            0
8679        );
8680    }
8681
8682    #[tokio::test]
8683    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
8684        let server = MockWorkerServer::start();
8685        let client = Client::builder(server.base_url())
8686            .timeout(Duration::from_secs(2))
8687            .build()
8688            .expect("client");
8689
8690        client
8691            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
8692            .await
8693            .expect("register");
8694        client
8695            .heartbeat_worker("capture-worker", 1, 1)
8696            .await
8697            .expect("heartbeat");
8698        client
8699            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
8700            .await
8701            .expect("workflow poll");
8702        client
8703            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
8704            .await
8705            .expect("activity poll");
8706
8707        for path in [
8708            "/api/worker/register",
8709            "/api/worker/heartbeat",
8710            "/api/worker/workflow-tasks/poll",
8711            "/api/worker/activity-tasks/poll",
8712        ] {
8713            assert_eq!(
8714                server.worker_protocol_for(path).as_deref(),
8715                Some(WORKER_PROTOCOL_VERSION),
8716                "unexpected protocol for {path}"
8717            );
8718        }
8719
8720        assert_eq!(
8721            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
8722            1
8723        );
8724        assert_eq!(
8725            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
8726            1
8727        );
8728    }
8729
8730    #[tokio::test]
8731    async fn query_task_endpoints_send_the_query_feature_protocol() {
8732        let server = MockWorkerServer::start();
8733        let client = Client::builder(server.base_url())
8734            .timeout(Duration::from_secs(2))
8735            .build()
8736            .expect("client");
8737
8738        client
8739            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8740            .await
8741            .expect("query poll");
8742        client
8743            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
8744            .await
8745            .expect("query complete");
8746        client
8747            .fail_query_task(
8748                "query-capture",
8749                "capture-worker",
8750                1,
8751                "failed",
8752                "query_rejected",
8753                "QueryFailed",
8754            )
8755            .await
8756            .expect("query fail");
8757
8758        for path in [
8759            "/api/worker/query-tasks/poll",
8760            "/api/worker/query-tasks/query-capture/complete",
8761            "/api/worker/query-tasks/query-capture/fail",
8762        ] {
8763            assert_eq!(
8764                server.worker_protocol_for(path).as_deref(),
8765                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
8766                "unexpected protocol for {path}"
8767            );
8768        }
8769
8770        assert_eq!(
8771            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
8772            1
8773        );
8774    }
8775
8776    #[tokio::test]
8777    async fn query_protocol_rejection_from_older_server_is_typed() {
8778        let server = MockWorkerServer::reject_query_protocol();
8779        let client = Client::builder(server.base_url())
8780            .timeout(Duration::from_secs(2))
8781            .build()
8782            .expect("client");
8783
8784        let error = client
8785            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8786            .await
8787            .expect_err("server below query protocol floor must reject");
8788        let Error::Protocol(failure) = error else {
8789            panic!("expected typed protocol failure");
8790        };
8791
8792        assert_eq!(failure.status, 400);
8793        assert_eq!(failure.reason, "unsupported_protocol_version");
8794        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
8795        assert_eq!(
8796            failure.requested_version.as_deref(),
8797            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8798        );
8799        assert_eq!(
8800            server
8801                .worker_protocol_for("/api/worker/query-tasks/poll")
8802                .as_deref(),
8803            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8804        );
8805    }
8806
8807    #[tokio::test]
8808    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
8809        let server = MockWorkerServer::reject_query_protocol();
8810        let client = Client::builder(server.base_url())
8811            .timeout(Duration::from_secs(2))
8812            .build()
8813            .expect("client");
8814        let mut worker = Worker::new(client, "rust-workers")
8815            .worker_id("baseline-worker")
8816            .poll_timeout(Duration::from_millis(10));
8817
8818        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
8819            Ok(Value::Null)
8820        });
8821
8822        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
8823        assert_eq!(
8824            server
8825                .worker_protocol_for("/api/worker/workflow-tasks/poll")
8826                .as_deref(),
8827            Some(WORKER_PROTOCOL_VERSION)
8828        );
8829        assert_eq!(
8830            server.worker_protocol_for("/api/worker/query-tasks/poll"),
8831            None,
8832            "a worker without query handlers must not use the query-task endpoint"
8833        );
8834    }
8835
8836    #[tokio::test]
8837    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
8838        let server = MockWorkerServer::reject_query_completion();
8839        let client = Client::builder(server.base_url())
8840            .timeout(Duration::from_secs(2))
8841            .build()
8842            .expect("client");
8843
8844        let error = client
8845            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
8846            .await
8847            .expect_err("expired completion must be rejected");
8848        let Error::QueryFailed(failure) = error else {
8849            panic!("expected typed query failure");
8850        };
8851        assert_eq!(failure.status, 409);
8852        assert_eq!(failure.reason, "query_task_timed_out");
8853
8854        let mut worker = Worker::new(client, "rust-workers")
8855            .worker_id("late-worker")
8856            .poll_timeout(Duration::from_millis(10));
8857        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8858        worker.register_query(
8859            "counter",
8860            "current",
8861            |_ctx, _args| async move { Ok(json!(8)) },
8862        );
8863
8864        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
8865        assert_eq!(
8866            worker
8867                .run_once()
8868                .await
8869                .expect("worker continues after late completion"),
8870            0
8871        );
8872        assert_eq!(
8873            server.request_count("/api/worker/query-tasks/query-late/complete"),
8874            2
8875        );
8876        assert_eq!(
8877            server.request_count("/api/worker/query-tasks/query-late/fail"),
8878            0,
8879            "a server completion rejection must not be reported as an encoding failure"
8880        );
8881    }
8882
8883    #[tokio::test]
8884    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
8885        let server = MockWorkerServer::start();
8886        let client = Client::builder(server.base_url())
8887            .timeout(Duration::from_secs(2))
8888            .build()
8889            .expect("client");
8890        let mut worker = Worker::new(client, "rust-workers")
8891            .worker_id("activity-only-worker")
8892            .poll_timeout(Duration::from_millis(10));
8893
8894        worker.register_activity(
8895            "activity.only",
8896            |_ctx, _args| async move { Ok(Value::Null) },
8897        );
8898
8899        worker.run_until(async {}).await.expect("run worker");
8900    }
8901
8902    #[tokio::test]
8903    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
8904        let server = MockWorkerServer::start();
8905        let client = Client::builder(server.base_url())
8906            .timeout(Duration::from_secs(2))
8907            .build()
8908            .expect("client");
8909        let mut worker = Worker::new(client, "rust-workers")
8910            .worker_id("workflow-only-worker")
8911            .poll_timeout(Duration::from_millis(10));
8912
8913        worker.register_workflow(
8914            "workflow.only",
8915            |_ctx, _input| async move { Ok(Value::Null) },
8916        );
8917
8918        worker.run_until(async {}).await.expect("run worker");
8919    }
8920
8921    #[tokio::test]
8922    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
8923        let server = MockWorkerServer::start();
8924        let client = Client::builder(server.base_url())
8925            .timeout(Duration::from_secs(2))
8926            .build()
8927            .expect("client");
8928        let observations = Arc::new(Mutex::new(Vec::new()));
8929        let observed = Arc::clone(&observations);
8930        let mut worker = Worker::new(client, "rust-workers")
8931            .worker_id("observed-heartbeat-worker")
8932            .poll_timeout(Duration::from_millis(10))
8933            .on_worker_heartbeat(move |observation| {
8934                observed
8935                    .lock()
8936                    .expect("heartbeat observations")
8937                    .push(observation.clone());
8938            });
8939
8940        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
8941            Ok(Value::Null)
8942        });
8943        worker
8944            .run_until(tokio::time::sleep(Duration::from_millis(20)))
8945            .await
8946            .expect("run worker");
8947
8948        let observations = observations.lock().expect("heartbeat observations");
8949        let first = observations.first().expect("heartbeat acknowledgement");
8950        assert_eq!(first.worker_id, "observed-heartbeat-worker");
8951        assert_eq!(first.task_queue, "rust-workers");
8952        assert!(first.acknowledged_at_unix_millis > 0);
8953        assert_eq!(first.acknowledgement, json!({}));
8954    }
8955
8956    #[tokio::test]
8957    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
8958        let server = MockWorkerServer::delayed_heartbeat_worker();
8959        let client = Client::builder(server.base_url())
8960            .timeout(Duration::from_secs(3))
8961            .build()
8962            .expect("client");
8963        let observations = Arc::new(Mutex::new(Vec::new()));
8964        let observed = Arc::clone(&observations);
8965        let mut worker = Worker::new(client, "rust-snapshot-workers")
8966            .worker_id("rust-snapshot-worker")
8967            .poll_timeout(Duration::from_millis(10))
8968            .on_worker_heartbeat(move |observation| {
8969                observed
8970                    .lock()
8971                    .expect("heartbeat observations")
8972                    .push(observation.clone());
8973            });
8974
8975        worker.register_workflow("snapshot", |ctx, _input| async move {
8976            ctx.wait_signal("finish").await?;
8977            Ok(json!({"status": "finished"}))
8978        });
8979        worker.register_query("snapshot", "current", |ctx, _args| async move {
8980            Ok(json!(ctx
8981                .signals("increment")
8982                .iter()
8983                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
8984                .sum::<i64>()))
8985        });
8986        worker.register_activity("cancel-aware", |_ctx, _args| async move {
8987            Ok(json!({"late": "completion"}))
8988        });
8989
8990        worker
8991            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
8992            .await
8993            .expect("delayed heartbeat must allow a clean worker shutdown");
8994
8995        let observations = observations.lock().expect("heartbeat observations");
8996        assert!(
8997            observations.len() >= 3,
8998            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
8999        );
9000        assert!(
9001            observations.windows(2).all(|pair| {
9002                pair[1].acknowledged_at_unix_millis
9003                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
9004                    >= 850
9005            }),
9006            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
9007        );
9008        drop(observations);
9009
9010        let heartbeat_times = server.request_times("/api/worker/heartbeat");
9011        let delayed_request_at = *heartbeat_times
9012            .get(1)
9013            .expect("intentionally delayed heartbeat request");
9014        let delay_window_start = delayed_request_at + Duration::from_millis(100);
9015        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
9016        for path in [
9017            "/api/worker/workflow-tasks/poll",
9018            "/api/worker/activity-tasks/poll",
9019            "/api/worker/query-tasks/poll",
9020        ] {
9021            assert!(
9022                server
9023                    .request_times(path)
9024                    .iter()
9025                    .any(|received_at| *received_at >= delay_window_start
9026                        && *received_at <= delay_window_end),
9027                "{path} must keep polling while a heartbeat acknowledgement is delayed"
9028            );
9029        }
9030        assert!(
9031            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
9032            "workflow work must be settled"
9033        );
9034        assert!(
9035            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
9036            "activity work must be settled"
9037        );
9038        assert!(
9039            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
9040            "query work must be settled"
9041        );
9042    }
9043
9044    #[tokio::test]
9045    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
9046        let server = MockWorkerServer::heartbeat_retry_worker();
9047        let client = Client::builder(server.base_url())
9048            .timeout(Duration::from_secs(2))
9049            .build()
9050            .expect("client");
9051        let observations = Arc::new(Mutex::new(Vec::new()));
9052        let observed = Arc::clone(&observations);
9053        let worker = Worker::new(client, "rust-workers")
9054            .worker_id("heartbeat-retry-worker")
9055            .retry_policy(WorkerRetryPolicy {
9056                max_retries: 1,
9057                initial_backoff: Duration::from_millis(300),
9058                max_backoff: Duration::from_millis(300),
9059            })
9060            .on_worker_heartbeat(move |observation| {
9061                observed
9062                    .lock()
9063                    .expect("heartbeat observations")
9064                    .push(observation.clone());
9065            });
9066
9067        worker
9068            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
9069            .await
9070            .expect("retryable heartbeat failure must remain bounded and recover");
9071
9072        let observations = observations.lock().expect("heartbeat observations");
9073        assert!(observations.len() >= 3, "heartbeat retry must recover");
9074        assert!(
9075            observations.windows(2).all(|pair| {
9076                pair[1]
9077                    .acknowledged_at_unix_millis
9078                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
9079                    >= 850
9080            }),
9081            "a successful retry must start a fresh advertised cadence: {observations:?}"
9082        );
9083        assert_eq!(
9084            server.request_count("/api/worker/heartbeat"),
9085            observations.len() + 1,
9086            "one retryable failure must add exactly one bounded request"
9087        );
9088    }
9089
9090    #[tokio::test]
9091    async fn query_enabled_worker_stays_live_when_signal_replay_emits_no_commands() {
9092        let server = MockWorkerServer::waiting_query_worker();
9093        let client = Client::builder(server.base_url())
9094            .timeout(Duration::from_secs(2))
9095            .build()
9096            .expect("client");
9097        let observations = Arc::new(Mutex::new(Vec::new()));
9098        let observed = Arc::clone(&observations);
9099        let mut worker = Worker::new(client, "rust-snapshot-workers")
9100            .worker_id("rust-snapshot-worker")
9101            .poll_timeout(Duration::from_millis(10))
9102            .on_worker_heartbeat(move |observation| {
9103                observed
9104                    .lock()
9105                    .expect("heartbeat observations")
9106                    .push(observation.clone());
9107            });
9108
9109        worker.register_workflow("snapshot", |ctx, _input| async move {
9110            ctx.wait_signal("finish").await?;
9111            Ok(json!({"status": "finished"}))
9112        });
9113        worker.register_query("snapshot", "current", |ctx, _args| async move {
9114            let current = ctx
9115                .signals("increment")
9116                .iter()
9117                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
9118                .sum::<i64>();
9119            Ok(json!(current))
9120        });
9121
9122        worker
9123            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
9124            .await
9125            .expect("pending workflow and query poller must remain live until shutdown");
9126
9127        assert!(
9128            observations.lock().expect("heartbeat observations").len() >= 4,
9129            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
9130        );
9131        assert!(
9132            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
9133            "workflow polling must continue after empty replay acknowledgements"
9134        );
9135        assert!(
9136            server.request_count("/api/worker/query-tasks/poll") >= 2,
9137            "query polling must continue after serving the current query"
9138        );
9139        assert_eq!(
9140            server.request_body("/api/worker/register")["capabilities"],
9141            json!([QUERY_TASKS_CAPABILITY])
9142        );
9143
9144        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
9145            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
9146            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
9147            let failure = server.request_body(&fail_path);
9148            assert_eq!(
9149                failure["failure"]["type"],
9150                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
9151            );
9152            assert_eq!(server.request_count(&completion_path), 0);
9153        }
9154
9155        let query_completion =
9156            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
9157        assert_eq!(query_completion["result"], json!(8));
9158    }
9159
9160    #[tokio::test]
9161    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
9162        let server = MockWorkerServer::transient_worker_failures();
9163        let client = Client::builder(server.base_url())
9164            .timeout(Duration::from_secs(2))
9165            .build()
9166            .expect("client");
9167        let mut worker = Worker::new(client, "rust-workers")
9168            .worker_id("retry-worker")
9169            .poll_timeout(Duration::from_millis(10))
9170            .retry_policy(WorkerRetryPolicy {
9171                max_retries: 2,
9172                initial_backoff: Duration::from_millis(1),
9173                max_backoff: Duration::from_millis(1),
9174            });
9175        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
9176        worker.register_activity(
9177            "counter.activity",
9178            |_ctx, _input| async move { Ok(Value::Null) },
9179        );
9180        worker.register_query(
9181            "counter",
9182            "current",
9183            |_ctx, _args| async move { Ok(json!(8)) },
9184        );
9185
9186        worker
9187            .run_until(tokio::time::sleep(Duration::from_millis(75)))
9188            .await
9189            .expect("transient failures must not stop the worker");
9190
9191        for path in [
9192            "/api/worker/heartbeat",
9193            "/api/worker/workflow-tasks/poll",
9194            "/api/worker/activity-tasks/poll",
9195            "/api/worker/query-tasks/poll",
9196        ] {
9197            assert!(
9198                server.request_count(path) >= 2,
9199                "{path} must continue after its transient failure"
9200            );
9201        }
9202    }
9203
9204    #[tokio::test]
9205    async fn worker_bounds_transport_retries() {
9206        let server = MockWorkerServer::unavailable_polls();
9207        let client = Client::builder(server.base_url())
9208            .timeout(Duration::from_secs(2))
9209            .build()
9210            .expect("client");
9211        let mut worker = Worker::new(client, "rust-workers")
9212            .worker_id("bounded-retry-worker")
9213            .poll_timeout(Duration::from_millis(10))
9214            .retry_policy(WorkerRetryPolicy {
9215                max_retries: 2,
9216                initial_backoff: Duration::from_millis(1),
9217                max_backoff: Duration::from_millis(1),
9218            });
9219        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
9220
9221        let error = worker.run().await.expect_err("retry bound must terminate");
9222        assert!(matches!(error, Error::Transport(_)));
9223        assert_eq!(
9224            server.request_count("/api/worker/workflow-tasks/poll"),
9225            3,
9226            "one initial request plus two retries"
9227        );
9228    }
9229
9230    #[tokio::test]
9231    async fn worker_does_not_retry_authentication_failures() {
9232        let server = MockWorkerServer::unauthorized_polls();
9233        let client = Client::builder(server.base_url())
9234            .timeout(Duration::from_secs(2))
9235            .build()
9236            .expect("client");
9237        let mut worker = Worker::new(client, "rust-workers")
9238            .worker_id("unauthorized-worker")
9239            .poll_timeout(Duration::from_millis(10));
9240        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
9241
9242        let error = worker
9243            .run()
9244            .await
9245            .expect_err("authentication must terminate");
9246        let Error::Http { status, body } = error else {
9247            panic!("expected stable HTTP authentication error");
9248        };
9249        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
9250        assert!(body.contains("authentication_failed"));
9251        assert_eq!(
9252            server.request_count("/api/worker/workflow-tasks/poll"),
9253            1,
9254            "authentication failures must not be retried"
9255        );
9256    }
9257
9258    #[derive(Clone, Debug)]
9259    struct CapturedRequest {
9260        path: String,
9261        worker_protocol: Option<String>,
9262        body: String,
9263        received_at: Instant,
9264    }
9265
9266    struct MockWorkerServer {
9267        addr: SocketAddr,
9268        stop: Arc<AtomicBool>,
9269        requests: Arc<Mutex<Vec<CapturedRequest>>>,
9270        thread: Option<thread::JoinHandle<()>>,
9271    }
9272
9273    #[derive(Clone, Copy, Default)]
9274    struct MockWorkerBehavior {
9275        reject_query_protocol: bool,
9276        reject_query_completion: bool,
9277        waiting_query_worker: bool,
9278        poll_failures_per_path: usize,
9279        heartbeat_failures: usize,
9280        heartbeat_failure_request: Option<usize>,
9281        delayed_heartbeat_request: Option<usize>,
9282        heartbeat_response_delay: Duration,
9283        concurrent_requests: bool,
9284        unauthorized_polls: bool,
9285        cancelled_activity: bool,
9286        draining_polls: bool,
9287    }
9288
9289    impl MockWorkerServer {
9290        fn start() -> Self {
9291            Self::start_with_behavior(MockWorkerBehavior::default())
9292        }
9293
9294        fn reject_query_protocol() -> Self {
9295            Self::start_with_behavior(MockWorkerBehavior {
9296                reject_query_protocol: true,
9297                ..MockWorkerBehavior::default()
9298            })
9299        }
9300
9301        fn reject_query_completion() -> Self {
9302            Self::start_with_behavior(MockWorkerBehavior {
9303                reject_query_completion: true,
9304                ..MockWorkerBehavior::default()
9305            })
9306        }
9307
9308        fn waiting_query_worker() -> Self {
9309            Self::start_with_behavior(MockWorkerBehavior {
9310                waiting_query_worker: true,
9311                ..MockWorkerBehavior::default()
9312            })
9313        }
9314
9315        fn transient_worker_failures() -> Self {
9316            Self::start_with_behavior(MockWorkerBehavior {
9317                poll_failures_per_path: 1,
9318                heartbeat_failures: 1,
9319                ..MockWorkerBehavior::default()
9320            })
9321        }
9322
9323        fn delayed_heartbeat_worker() -> Self {
9324            Self::start_with_behavior(MockWorkerBehavior {
9325                waiting_query_worker: true,
9326                delayed_heartbeat_request: Some(2),
9327                heartbeat_response_delay: Duration::from_millis(1_500),
9328                concurrent_requests: true,
9329                cancelled_activity: true,
9330                ..MockWorkerBehavior::default()
9331            })
9332        }
9333
9334        fn heartbeat_retry_worker() -> Self {
9335            Self::start_with_behavior(MockWorkerBehavior {
9336                waiting_query_worker: true,
9337                heartbeat_failure_request: Some(2),
9338                concurrent_requests: true,
9339                ..MockWorkerBehavior::default()
9340            })
9341        }
9342
9343        fn unavailable_polls() -> Self {
9344            Self::start_with_behavior(MockWorkerBehavior {
9345                poll_failures_per_path: usize::MAX,
9346                ..MockWorkerBehavior::default()
9347            })
9348        }
9349
9350        fn unauthorized_polls() -> Self {
9351            Self::start_with_behavior(MockWorkerBehavior {
9352                unauthorized_polls: true,
9353                ..MockWorkerBehavior::default()
9354            })
9355        }
9356
9357        fn cancelled_activity() -> Self {
9358            Self::start_with_behavior(MockWorkerBehavior {
9359                cancelled_activity: true,
9360                ..MockWorkerBehavior::default()
9361            })
9362        }
9363
9364        fn draining_polls() -> Self {
9365            Self::start_with_behavior(MockWorkerBehavior {
9366                draining_polls: true,
9367                ..MockWorkerBehavior::default()
9368            })
9369        }
9370
9371        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
9372            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
9373            listener
9374                .set_nonblocking(true)
9375                .expect("configure mock listener");
9376            let addr = listener.local_addr().expect("mock server address");
9377            let stop = Arc::new(AtomicBool::new(false));
9378            let server_stop = Arc::clone(&stop);
9379            let requests = Arc::new(Mutex::new(Vec::new()));
9380            let server_requests = Arc::clone(&requests);
9381            let thread = thread::spawn(move || {
9382                let mut request_threads = Vec::new();
9383                while !server_stop.load(Ordering::SeqCst) {
9384                    match listener.accept() {
9385                        Ok((mut stream, _)) => {
9386                            if behavior.concurrent_requests {
9387                                let requests = Arc::clone(&server_requests);
9388                                request_threads.push(thread::spawn(move || {
9389                                    handle_mock_worker_request(&mut stream, &requests, behavior)
9390                                }));
9391                            } else {
9392                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
9393                            }
9394                        }
9395                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
9396                            let mut index = 0;
9397                            while index < request_threads.len() {
9398                                if request_threads[index].is_finished() {
9399                                    request_threads
9400                                        .swap_remove(index)
9401                                        .join()
9402                                        .expect("join mock request");
9403                                } else {
9404                                    index += 1;
9405                                }
9406                            }
9407                            thread::sleep(Duration::from_millis(5));
9408                        }
9409                        Err(_) => break,
9410                    }
9411                }
9412                for request_thread in request_threads {
9413                    request_thread.join().expect("join mock request");
9414                }
9415            });
9416
9417            Self {
9418                addr,
9419                stop,
9420                requests,
9421                thread: Some(thread),
9422            }
9423        }
9424
9425        fn base_url(&self) -> String {
9426            format!("http://{}", self.addr)
9427        }
9428
9429        fn worker_protocol_for(&self, path: &str) -> Option<String> {
9430            self.requests
9431                .lock()
9432                .expect("captured requests")
9433                .iter()
9434                .find(|request| request.path == path)
9435                .and_then(|request| request.worker_protocol.clone())
9436        }
9437
9438        fn request_count(&self, path: &str) -> usize {
9439            self.requests
9440                .lock()
9441                .expect("captured requests")
9442                .iter()
9443                .filter(|request| request.path == path)
9444                .count()
9445        }
9446
9447        fn request_times(&self, path: &str) -> Vec<Instant> {
9448            self.requests
9449                .lock()
9450                .expect("captured requests")
9451                .iter()
9452                .filter(|request| request.path == path)
9453                .map(|request| request.received_at)
9454                .collect()
9455        }
9456
9457        fn request_body(&self, path: &str) -> Value {
9458            let requests = self.requests.lock().expect("captured requests");
9459            let body = &requests
9460                .iter()
9461                .find(|request| request.path == path)
9462                .unwrap_or_else(|| panic!("missing request for {path}"))
9463                .body;
9464            serde_json::from_str(body).unwrap_or_else(|error| {
9465                panic!("invalid JSON request body for {path}: {error}: {body:?}")
9466            })
9467        }
9468    }
9469
9470    impl Drop for MockWorkerServer {
9471        fn drop(&mut self) {
9472            self.stop.store(true, Ordering::SeqCst);
9473            let _ = TcpStream::connect(self.addr);
9474
9475            if let Some(thread) = self.thread.take() {
9476                thread.join().expect("join mock server");
9477            }
9478        }
9479    }
9480
9481    fn handle_mock_worker_request(
9482        stream: &mut TcpStream,
9483        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
9484        behavior: MockWorkerBehavior,
9485    ) {
9486        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
9487        let mut buffer = [0_u8; 8192];
9488        let mut request = Vec::new();
9489
9490        loop {
9491            match stream.read(&mut buffer) {
9492                Ok(0) => break,
9493                Ok(read) => {
9494                    request.extend_from_slice(&buffer[..read]);
9495                    if mock_request_is_complete(&request) {
9496                        break;
9497                    }
9498                }
9499                Err(error)
9500                    if matches!(
9501                        error.kind(),
9502                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
9503                    ) =>
9504                {
9505                    break;
9506                }
9507                Err(_) => return,
9508            }
9509        }
9510
9511        let request = String::from_utf8_lossy(&request);
9512        let body = request
9513            .split_once("\r\n\r\n")
9514            .map(|(_, body)| body)
9515            .unwrap_or_default();
9516        let path = request
9517            .lines()
9518            .next()
9519            .and_then(|line| line.split_whitespace().nth(1))
9520            .unwrap_or_default();
9521        let worker_protocol = request.lines().find_map(|line| {
9522            let (name, value) = line.split_once(':')?;
9523            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
9524                .then(|| value.trim().to_string())
9525        });
9526        let request_number = {
9527            let mut requests = requests.lock().expect("captured requests");
9528            requests.push(CapturedRequest {
9529                path: path.to_string(),
9530                worker_protocol: worker_protocol.clone(),
9531                body: body.to_string(),
9532                received_at: Instant::now(),
9533            });
9534            requests
9535                .iter()
9536                .filter(|request| request.path == path)
9537                .count()
9538        };
9539
9540        let is_poll = matches!(
9541            path,
9542            "/api/worker/workflow-tasks/poll"
9543                | "/api/worker/activity-tasks/poll"
9544                | "/api/worker/query-tasks/poll"
9545        );
9546        if is_poll && request_number <= behavior.poll_failures_per_path {
9547            return;
9548        }
9549        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
9550            return;
9551        }
9552        if path == "/api/worker/heartbeat"
9553            && behavior.heartbeat_failure_request == Some(request_number)
9554        {
9555            return;
9556        }
9557        if path == "/api/worker/heartbeat"
9558            && behavior.delayed_heartbeat_request == Some(request_number)
9559        {
9560            thread::sleep(behavior.heartbeat_response_delay);
9561        }
9562        if behavior.unauthorized_polls && is_poll {
9563            write_mock_response(
9564                stream,
9565                "401 Unauthorized",
9566                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
9567            );
9568            return;
9569        }
9570        if behavior.draining_polls && is_poll {
9571            write_mock_response(
9572                stream,
9573                "409 Conflict",
9574                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
9575            );
9576            return;
9577        }
9578
9579        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
9580            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
9581            let body = format!(
9582                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
9583            );
9584            write_mock_response(stream, "400 Bad Request", &body);
9585            return;
9586        }
9587
9588        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
9589        {
9590            write_mock_response(
9591                stream,
9592                "409 Conflict",
9593                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
9594            );
9595            return;
9596        }
9597
9598        if behavior.waiting_query_worker {
9599            if path == "/api/worker/workflow-tasks/poll" && request_number <= 2 {
9600                let amounts = if request_number == 1 {
9601                    vec![3]
9602                } else {
9603                    vec![3, 5]
9604                };
9605                let task_id = if request_number == 1 {
9606                    "snapshot-wait-3"
9607                } else {
9608                    "snapshot-wait-5"
9609                };
9610                let history_events = std::iter::once(json!({
9611                    "event_type": "SignalWaitOpened",
9612                    "payload": {"sequence": 1, "signal_name": "finish"}
9613                }))
9614                .chain(amounts.iter().enumerate().map(|(index, amount)| {
9615                    json!({
9616                        "event_type": "SignalReceived",
9617                        "payload": {
9618                            "signal_id": format!("increment-{amount}"),
9619                            "signal_name": "increment",
9620                            "workflow_sequence": index + 2,
9621                            "payload_codec": DEFAULT_CODEC,
9622                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9623                                .expect("Avro signal envelope")
9624                        }
9625                    })
9626                }))
9627                .collect::<Vec<_>>();
9628                let body = json!({
9629                    "task": {
9630                        "task_id": task_id,
9631                        "workflow_id": "snapshot-1",
9632                        "run_id": "snapshot-run-1",
9633                        "workflow_type": "snapshot",
9634                        "payload_codec": DEFAULT_CODEC,
9635                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9636                            .expect("Avro workflow arguments"),
9637                        "history_events": history_events,
9638                        "workflow_task_attempt": 1,
9639                        "workflow_signal_id": format!("increment-{}", amounts.last().expect("amount")),
9640                        "signal_name": "increment",
9641                        "signal_arguments": encode_value_envelope(
9642                            &json!([amounts.last().expect("amount")]),
9643                            DEFAULT_CODEC,
9644                        )
9645                        .expect("Avro resume signal"),
9646                        "lease_owner": "rust-snapshot-worker"
9647                    }
9648                })
9649                .to_string();
9650                write_mock_response(stream, "200 OK", &body);
9651                return;
9652            }
9653
9654            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
9655                let history_events = [3, 5]
9656                    .into_iter()
9657                    .enumerate()
9658                    .map(|(index, amount)| {
9659                        json!({
9660                            "event_type": "SignalReceived",
9661                            "payload": {
9662                                "signal_id": format!("increment-{amount}"),
9663                                "signal_name": "increment",
9664                                "workflow_sequence": index + 2,
9665                                "payload_codec": DEFAULT_CODEC,
9666                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9667                                    .expect("Avro query signal envelope")
9668                            }
9669                        })
9670                    })
9671                    .collect::<Vec<_>>();
9672                let body = json!({
9673                    "task": {
9674                        "query_task_id": "snapshot-current",
9675                        "query_task_attempt": 1,
9676                        "lease_owner": "rust-snapshot-worker",
9677                        "workflow_id": "snapshot-1",
9678                        "run_id": "snapshot-run-1",
9679                        "workflow_type": "snapshot",
9680                        "query_name": "current",
9681                        "payload_codec": DEFAULT_CODEC,
9682                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9683                            .expect("Avro workflow arguments"),
9684                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9685                            .expect("Avro query arguments"),
9686                        "history_events": history_events,
9687                        "run_status": "waiting"
9688                    }
9689                })
9690                .to_string();
9691                write_mock_response(stream, "200 OK", &body);
9692                return;
9693            }
9694
9695            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
9696                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
9697            {
9698                write_mock_response(
9699                    stream,
9700                    "200 OK",
9701                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
9702                );
9703                return;
9704            }
9705
9706            if path == "/api/worker/query-tasks/snapshot-current/complete" {
9707                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
9708                return;
9709            }
9710        }
9711
9712        let (status, body) = match path {
9713            "/api/workflows" => (
9714                "201 Created",
9715                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
9716            ),
9717            "/api/worker/register" if behavior.waiting_query_worker => (
9718                "200 OK",
9719                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
9720            ),
9721            "/api/worker/register" => (
9722                "200 OK",
9723                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
9724            ),
9725            "/api/worker/heartbeat" => ("200 OK", "{}"),
9726            "/api/worker/activity-tasks/poll"
9727                if behavior.cancelled_activity && request_number == 1 =>
9728            {
9729                (
9730                    "200 OK",
9731                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
9732                )
9733            }
9734            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
9735                ("200 OK", r#"{"task":null}"#)
9736            }
9737            "/api/worker/query-tasks/poll"
9738                if behavior.reject_query_completion && request_number == 1 =>
9739            {
9740                (
9741                    "200 OK",
9742                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
9743                )
9744            }
9745            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
9746            "/api/worker/query-tasks/query-capture/complete"
9747            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
9748            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
9749                "200 OK",
9750                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
9751            ),
9752            "/api/worker/activity-tasks/activity-cancel/complete" => (
9753                "409 Conflict",
9754                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
9755            ),
9756            "/api/workflows/counter-1/query/current" => (
9757                "200 OK",
9758                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
9759            ),
9760            "/api/workflows/counter-1/query/missing" => (
9761                "404 Not Found",
9762                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
9763            ),
9764            "/api/workflows/wf-lifecycle/cancel" => (
9765                "200 OK",
9766                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
9767            ),
9768            "/api/workflows/wf-lifecycle/terminate" => (
9769                "200 OK",
9770                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
9771            ),
9772            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
9773                "200 OK",
9774                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
9775            ),
9776            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
9777                "200 OK",
9778                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
9779            ),
9780            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
9781            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
9782                "409 Conflict",
9783                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
9784            ),
9785            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
9786                "200 OK",
9787                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
9788            ),
9789            "/api/workflows/wf-cancelled" => (
9790                "200 OK",
9791                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
9792            ),
9793            "/api/workflows/wf-terminated" => (
9794                "200 OK",
9795                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
9796            ),
9797            "/api/workflows/wf-timed-out" => (
9798                "200 OK",
9799                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
9800            ),
9801            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
9802                "200 OK",
9803                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
9804            ),
9805            "/api/workflows/wf-selected" => (
9806                "200 OK",
9807                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
9808            ),
9809            "/api/workflows/wf-selected/runs/run-selected" => (
9810                "200 OK",
9811                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
9812            ),
9813            _ => ("404 Not Found", r#"{"message":"not found"}"#),
9814        };
9815        write_mock_response(stream, status, body);
9816    }
9817
9818    fn mock_request_is_complete(request: &[u8]) -> bool {
9819        let Some(header_end) = request
9820            .windows(4)
9821            .position(|window| window == b"\r\n\r\n")
9822            .map(|position| position + 4)
9823        else {
9824            return false;
9825        };
9826        let headers = String::from_utf8_lossy(&request[..header_end]);
9827        let content_length = headers.lines().find_map(|line| {
9828            let (name, value) = line.split_once(':')?;
9829            name.eq_ignore_ascii_case("content-length")
9830                .then(|| value.trim().parse::<usize>().ok())
9831                .flatten()
9832        });
9833
9834        request.len() >= header_end + content_length.unwrap_or(0)
9835    }
9836
9837    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
9838        let response = format!(
9839            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
9840            body.len()
9841        );
9842
9843        let _ = stream.write_all(response.as_bytes());
9844        let _ = stream.flush();
9845    }
9846}