Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21    de::DeserializeOwned,
22    ser::{SerializeMap, SerializeSeq},
23    Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34/// Worker-registration capability for server-routed read-only queries.
35pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36/// Worker-registration capability for synchronous workflow updates.
37pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38/// First additive worker protocol that defines query-task transport.
39pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43    "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47    "lease_expired",
48    "query_task_not_found",
49    "query_task_not_leased",
50    "query_task_timed_out",
51];
52
53/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
54pub const AVRO_VALUE_SCHEMA_JSON: &str =
55    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64    ControlPlane,
65    Worker(&'static str),
66}
67
68impl RequestProtocol {
69    fn is_worker(self) -> bool {
70        matches!(self, Self::Worker(_))
71    }
72}
73
74pub type Result<T> = std::result::Result<T, Error>;
75
76#[derive(Debug, Error)]
77pub enum Error {
78    #[error("transport error: {0}")]
79    Transport(#[from] reqwest::Error),
80    #[error(
81        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
82    )]
83    InvalidBaseUrl,
84    #[error("json error: {0}")]
85    Json(#[from] serde_json::Error),
86    #[error("http {status}: {body}")]
87    Http {
88        status: reqwest::StatusCode,
89        body: String,
90    },
91    #[error("codec error: {0}")]
92    Codec(String),
93    #[error(transparent)]
94    QueryFailed(QueryFailure),
95    #[error(transparent)]
96    Protocol(ProtocolFailure),
97    #[error(transparent)]
98    NonDeterministicReplay(ReplayFailure),
99    #[error(transparent)]
100    ChildWorkflowFailed(ChildWorkflowFailure),
101    #[error(transparent)]
102    ActivityFailed(ActivityFailure),
103    #[error(transparent)]
104    WorkflowCommandRejected(WorkflowCommandRejection),
105    #[error(transparent)]
106    WorkflowFailed(WorkflowTerminalOutcome),
107    #[error(transparent)]
108    WorkflowCancelled(WorkflowTerminalOutcome),
109    #[error(transparent)]
110    WorkflowTerminated(WorkflowTerminalOutcome),
111    #[error(transparent)]
112    WorkflowTimedOut(WorkflowTerminalOutcome),
113    #[error(transparent)]
114    ActivityTaskRejected(ActivityTaskRejection),
115    #[error("workflow handler {0:?} is not registered")]
116    WorkflowNotRegistered(String),
117    #[error("activity handler {0:?} is not registered")]
118    ActivityNotRegistered(String),
119    #[error("workflow future yielded without emitting a durable command")]
120    WorkflowYieldedWithoutCommand,
121    #[error("workflow state lock is poisoned")]
122    WorkflowStatePoisoned,
123    #[error("timer duration is too large for the worker protocol")]
124    TimerDurationOverflow,
125    #[error("operation timed out")]
126    Timeout,
127    #[error("worker loop error: {0}")]
128    WorkerLoop(String),
129    #[error("invalid child workflow options: {0}")]
130    InvalidChildWorkflowOptions(String),
131    #[error(transparent)]
132    InvalidActivityOptions(ActivityOptionsError),
133    #[error(transparent)]
134    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
135    #[doc(hidden)]
136    #[error("workflow requested continue as new")]
137    ContinueAsNew(ContinueAsNewRequest),
138}
139
140/// The lifecycle command sent to a workflow execution.
141#[derive(Clone, Copy, Debug, PartialEq, Eq)]
142pub enum WorkflowCommandKind {
143    Cancel,
144    Terminate,
145}
146
147impl WorkflowCommandKind {
148    fn as_str(self) -> &'static str {
149        match self {
150            Self::Cancel => "cancel",
151            Self::Terminate => "terminate",
152        }
153    }
154}
155
156/// Optional structured fields for a cancellation or termination request.
157#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
158pub struct WorkflowCommandOptions {
159    #[serde(skip_serializing_if = "Option::is_none")]
160    pub reason: Option<String>,
161    #[serde(skip_serializing_if = "Option::is_none")]
162    pub request_id: Option<String>,
163}
164
165/// Server-enforced timeout policy for a workflow start.
166///
167/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
168/// only bounds how long the caller waits. A server deadline produces a terminal
169/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
170/// `run_timeout`.
171#[derive(Clone, Debug, PartialEq, Eq)]
172pub struct WorkflowStartOptions {
173    pub execution_timeout_seconds: u64,
174    pub run_timeout_seconds: u64,
175}
176
177impl Default for WorkflowStartOptions {
178    fn default() -> Self {
179        Self {
180            execution_timeout_seconds: 3600,
181            run_timeout_seconds: 600,
182        }
183    }
184}
185
186impl WorkflowStartOptions {
187    pub fn new() -> Self {
188        Self::default()
189    }
190
191    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
192        self.execution_timeout_seconds = seconds;
193        self
194    }
195
196    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
197        self.run_timeout_seconds = seconds;
198        self
199    }
200
201    fn validate(&self) -> Result<()> {
202        if self.execution_timeout_seconds == 0 {
203            return Err(Error::Codec(
204                "execution_timeout_seconds must be at least 1".to_string(),
205            ));
206        }
207        if self.run_timeout_seconds == 0 {
208            return Err(Error::Codec(
209                "run_timeout_seconds must be at least 1".to_string(),
210            ));
211        }
212        if self.run_timeout_seconds > self.execution_timeout_seconds {
213            return Err(Error::Codec(
214                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
215            ));
216        }
217
218        Ok(())
219    }
220}
221
222/// Optional routing overrides for a continue-as-new transition.
223///
224/// Omitted values retain the current workflow type and task queue. Server-owned
225/// instance metadata is not accepted here and is carried by the server.
226#[derive(Clone, Debug, Default, PartialEq, Eq)]
227pub struct ContinueAsNewOptions {
228    pub workflow_type: Option<String>,
229    pub task_queue: Option<String>,
230}
231
232impl ContinueAsNewOptions {
233    pub fn new() -> Self {
234        Self::default()
235    }
236
237    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
238        self.workflow_type = Some(workflow_type.into());
239        self
240    }
241
242    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
243        self.task_queue = Some(task_queue.into());
244        self
245    }
246
247    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
248        for (field, value) in [
249            ("workflow_type", self.workflow_type.as_deref()),
250            ("task_queue", self.task_queue.as_deref()),
251        ] {
252            if value.is_some_and(|value| value.trim().is_empty()) {
253                return Err(ContinueAsNewOptionsError {
254                    field,
255                    message: format!("{field} must not be empty"),
256                });
257            }
258        }
259        Ok(())
260    }
261}
262
263/// A stable validation error raised before a continue-as-new command is emitted.
264#[derive(Clone, Debug, Error, PartialEq, Eq)]
265#[error("invalid continue-as-new option {field}: {message}")]
266pub struct ContinueAsNewOptionsError {
267    pub field: &'static str,
268    pub message: String,
269}
270
271/// Public history-budget information attached to the current workflow task.
272#[derive(Clone, Debug, Default, PartialEq, Eq)]
273pub struct WorkflowHistoryBudget {
274    pub event_count: u64,
275    pub size_bytes: Option<u64>,
276    pub continue_as_new_recommended: bool,
277    pub pressure: Option<String>,
278}
279
280#[doc(hidden)]
281#[derive(Clone, Debug)]
282pub struct ContinueAsNewRequest {
283    arguments: AvroValue,
284    options: ContinueAsNewOptions,
285}
286
287impl WorkflowCommandOptions {
288    pub fn new() -> Self {
289        Self::default()
290    }
291
292    pub fn reason(mut self, reason: impl Into<String>) -> Self {
293        self.reason = Some(reason.into());
294        self
295    }
296
297    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
298        self.request_id = Some(request_id.into());
299        self
300    }
301}
302
303/// The accepted, machine-readable result of a lifecycle command.
304#[derive(Clone, Debug, PartialEq)]
305pub struct WorkflowCommandResult {
306    pub command: WorkflowCommandKind,
307    pub workflow_id: String,
308    pub run_id: Option<String>,
309    pub outcome: Option<String>,
310    pub reason: Option<String>,
311    pub command_status: Option<String>,
312    pub raw: Value,
313}
314
315/// A stable rejection returned by instance- or selected-run lifecycle commands.
316#[derive(Clone, Debug, Error)]
317#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
318pub struct WorkflowCommandRejection {
319    pub command: WorkflowCommandKind,
320    pub status: u16,
321    pub reason: String,
322    pub message: String,
323    pub workflow_id: String,
324    pub run_id: Option<String>,
325    pub target_scope: Option<String>,
326    pub body: Value,
327}
328
329/// Stable terminal categories returned by [`WorkflowHandle::result`].
330#[derive(Clone, Copy, Debug, PartialEq, Eq)]
331pub enum WorkflowTerminalKind {
332    Failed,
333    Cancelled,
334    Terminated,
335    TimedOut,
336}
337
338/// A typed terminal workflow outcome with durable identity and failure metadata.
339///
340/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
341/// of parsing its display representation. Fields remain `None` when an older
342/// server did not publish that metadata.
343#[derive(Clone, Debug, Error)]
344#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
345pub struct WorkflowTerminalOutcome {
346    pub kind: WorkflowTerminalKind,
347    pub workflow_id: String,
348    pub run_id: Option<String>,
349    pub reason: String,
350    pub failure_category: Option<String>,
351    pub failure_id: Option<String>,
352    pub exception_type: Option<String>,
353    pub exception_class: Option<String>,
354    pub non_retryable: Option<bool>,
355    pub message: Option<String>,
356    pub exception: Option<Value>,
357    pub raw: Value,
358}
359
360/// A worker-side activity settlement or heartbeat rejected by durable state.
361#[derive(Clone, Debug, Error)]
362#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
363pub struct ActivityTaskRejection {
364    pub operation: String,
365    pub status: u16,
366    pub reason: String,
367    pub task_id: String,
368    pub activity_attempt_id: String,
369    pub cancel_requested: bool,
370    pub can_continue: Option<bool>,
371    pub run_closed_reason: Option<String>,
372    pub body: Value,
373}
374
375/// Stable validation categories for [`ActivityOptions`].
376#[derive(Clone, Copy, Debug, PartialEq, Eq)]
377pub enum ActivityOptionsErrorKind {
378    EmptyTaskQueue,
379    EmptyRetryPolicy,
380    InvalidMaxAttempts,
381    BackoffWithoutRetryBudget,
382    TooManyBackoffIntervals,
383    InvalidBackoffCoefficient,
384    BackoffGenerationTooLarge,
385    BackoffOverflow,
386    EmptyNonRetryableErrorType,
387    TimeoutNotPositive,
388    TimeoutOverflow,
389    TimeoutOrder,
390}
391
392/// A machine-readable activity-options validation failure.
393#[derive(Clone, Debug, Error, PartialEq, Eq)]
394#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
395pub struct ActivityOptionsError {
396    pub kind: ActivityOptionsErrorKind,
397    pub field: Option<&'static str>,
398    pub message: String,
399}
400
401impl ActivityOptionsError {
402    fn new(
403        kind: ActivityOptionsErrorKind,
404        field: Option<&'static str>,
405        message: impl Into<String>,
406    ) -> Self {
407        Self {
408            kind,
409            field,
410            message: message.into(),
411        }
412    }
413}
414
415/// Stable terminal categories returned when an awaited activity does not succeed.
416#[derive(Clone, Copy, Debug, PartialEq, Eq)]
417pub enum ActivityFailureKind {
418    Failed,
419    Cancelled,
420    TimedOut,
421}
422
423/// A stable, machine-readable terminal activity failure.
424///
425/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
426/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
427#[derive(Clone, Debug, Error)]
428#[error("activity failed ({reason}): {message}")]
429pub struct ActivityFailure {
430    pub kind: ActivityFailureKind,
431    pub reason: String,
432    pub message: String,
433    pub activity_execution_id: Option<String>,
434    pub activity_attempt_id: Option<String>,
435    pub activity_type: Option<String>,
436    pub activity_class: Option<String>,
437    pub attempt_number: Option<u64>,
438    pub failure_id: Option<String>,
439    pub failure_category: Option<String>,
440    pub timeout_kind: Option<String>,
441    pub non_retryable: bool,
442    pub exception_type: Option<String>,
443    pub exception_class: Option<String>,
444    pub code: Option<Value>,
445    pub exception: Option<Value>,
446}
447
448/// Stable terminal categories returned when an awaited child does not succeed.
449#[derive(Clone, Copy, Debug, PartialEq, Eq)]
450pub enum ChildWorkflowFailureKind {
451    Failed,
452    Cancelled,
453    Terminated,
454}
455
456/// A stable, machine-readable child workflow failure delivered to its parent.
457///
458/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
459/// of parsing the display message. Child and parent identifiers retain the
460/// relationship recorded in durable history across worker restarts.
461#[derive(Clone, Debug, Error)]
462#[error("child workflow failed ({reason}): {message}")]
463pub struct ChildWorkflowFailure {
464    pub kind: ChildWorkflowFailureKind,
465    pub reason: String,
466    pub message: String,
467    pub parent_workflow_id: Option<String>,
468    pub parent_workflow_run_id: Option<String>,
469    pub child_workflow_id: Option<String>,
470    pub child_workflow_run_id: Option<String>,
471    pub child_workflow_type: Option<String>,
472    pub failure_id: Option<String>,
473    pub failure_category: Option<String>,
474    pub exception_type: Option<String>,
475    pub exception_class: Option<String>,
476    pub non_retryable: bool,
477    pub code: Option<Value>,
478    pub exception: Option<Value>,
479}
480
481/// The identity of one durable workflow execution.
482#[derive(Clone, Debug, PartialEq, Eq)]
483pub struct WorkflowIdentity {
484    pub workflow_id: Option<String>,
485    pub run_id: Option<String>,
486}
487
488/// A successful child result together with its durable parent-child identity.
489#[derive(Clone, Debug, PartialEq)]
490pub struct ChildWorkflowResult {
491    pub parent: WorkflowIdentity,
492    pub child: WorkflowIdentity,
493    pub child_workflow_type: Option<String>,
494    pub result: Value,
495}
496
497/// Lossless successful child result for fixed Avro Value workflows.
498#[derive(Clone, Debug, PartialEq)]
499pub struct ChildWorkflowAvroResult {
500    pub parent: WorkflowIdentity,
501    pub child: WorkflowIdentity,
502    pub child_workflow_type: Option<String>,
503    pub result: AvroValue,
504}
505
506/// Server behavior when a parent closes while its child is still open.
507#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
508pub enum ParentClosePolicy {
509    #[default]
510    Abandon,
511    RequestCancel,
512    Terminate,
513}
514
515impl ParentClosePolicy {
516    fn as_str(self) -> &'static str {
517        match self {
518            Self::Abandon => "abandon",
519            Self::RequestCancel => "request_cancel",
520            Self::Terminate => "terminate",
521        }
522    }
523}
524
525/// Durable retry policy for one child workflow invocation.
526#[derive(Clone, Debug, Default, PartialEq, Eq)]
527pub struct ChildWorkflowRetryPolicy {
528    pub max_attempts: Option<u32>,
529    pub backoff_seconds: Vec<u64>,
530    pub non_retryable_error_types: Vec<String>,
531}
532
533/// Options recorded with a child-workflow command.
534///
535/// The task queue is mandatory so routing is explicit and replay-stable.
536#[derive(Clone, Debug, PartialEq, Eq)]
537pub struct ChildWorkflowOptions {
538    pub task_queue: String,
539    pub parent_close_policy: ParentClosePolicy,
540    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
541    pub execution_timeout_seconds: Option<u64>,
542    pub run_timeout_seconds: Option<u64>,
543}
544
545impl ChildWorkflowOptions {
546    pub fn new(task_queue: impl Into<String>) -> Self {
547        Self {
548            task_queue: task_queue.into(),
549            parent_close_policy: ParentClosePolicy::Abandon,
550            retry_policy: None,
551            execution_timeout_seconds: None,
552            run_timeout_seconds: None,
553        }
554    }
555
556    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
557        self.parent_close_policy = policy;
558        self
559    }
560
561    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
562        self.retry_policy = Some(policy);
563        self
564    }
565
566    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
567        self.execution_timeout_seconds = Some(seconds);
568        self
569    }
570
571    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
572        self.run_timeout_seconds = Some(seconds);
573        self
574    }
575}
576
577/// Backoff intervals for one durable activity retry policy.
578#[derive(Clone, Debug, PartialEq, Eq)]
579pub enum ActivityBackoff {
580    /// Use these intervals between attempts. The server repeats the final
581    /// interval if the retry budget contains more attempts than entries.
582    Explicit(Vec<Duration>),
583    /// Generate one interval for every retry using integer exponential growth.
584    Exponential {
585        initial_interval: Duration,
586        coefficient: u32,
587        maximum_interval: Option<Duration>,
588    },
589}
590
591/// Durable server-side retry policy for one activity execution.
592#[derive(Clone, Debug, Default, PartialEq, Eq)]
593pub struct ActivityRetryPolicy {
594    pub max_attempts: Option<u32>,
595    pub backoff: Option<ActivityBackoff>,
596    pub non_retryable_error_types: Vec<String>,
597}
598
599impl ActivityRetryPolicy {
600    /// Start a policy with a finite attempt budget, including the first attempt.
601    pub fn new(max_attempts: u32) -> Self {
602        Self {
603            max_attempts: Some(max_attempts),
604            ..Self::default()
605        }
606    }
607
608    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
609        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
610        self
611    }
612
613    pub fn exponential_backoff(
614        mut self,
615        initial_interval: Duration,
616        coefficient: u32,
617        maximum_interval: Option<Duration>,
618    ) -> Self {
619        self.backoff = Some(ActivityBackoff::Exponential {
620            initial_interval,
621            coefficient,
622            maximum_interval,
623        });
624        self
625    }
626
627    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
628        self.non_retryable_error_types.push(error_type.into());
629        self
630    }
631
632    pub fn non_retryable_error_types(
633        mut self,
634        error_types: impl IntoIterator<Item = impl Into<String>>,
635    ) -> Self {
636        self.non_retryable_error_types
637            .extend(error_types.into_iter().map(Into::into));
638        self
639    }
640}
641
642/// Options recorded atomically on one deterministic `schedule_activity` command.
643///
644/// Durations are rounded up to whole seconds when encoded, so the server never
645/// receives a shorter timeout or backoff than the caller requested.
646#[derive(Clone, Debug, Default, PartialEq, Eq)]
647pub struct ActivityOptions {
648    pub task_queue: Option<String>,
649    pub retry_policy: Option<ActivityRetryPolicy>,
650    pub start_to_close_timeout: Option<Duration>,
651    pub schedule_to_start_timeout: Option<Duration>,
652    pub schedule_to_close_timeout: Option<Duration>,
653    pub heartbeat_timeout: Option<Duration>,
654}
655
656impl ActivityOptions {
657    pub fn new() -> Self {
658        Self::default()
659    }
660
661    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
662        self.task_queue = Some(task_queue.into());
663        self
664    }
665
666    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
667        self.retry_policy = Some(policy);
668        self
669    }
670
671    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
672        self.start_to_close_timeout = Some(timeout);
673        self
674    }
675
676    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
677        self.schedule_to_start_timeout = Some(timeout);
678        self
679    }
680
681    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
682        self.schedule_to_close_timeout = Some(timeout);
683        self
684    }
685
686    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
687        self.heartbeat_timeout = Some(timeout);
688        self
689    }
690
691    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
692        if self
693            .task_queue
694            .as_deref()
695            .is_some_and(|queue| queue.trim().is_empty())
696        {
697            return Err(ActivityOptionsError::new(
698                ActivityOptionsErrorKind::EmptyTaskQueue,
699                Some("task_queue"),
700                "task_queue must not be empty",
701            ));
702        }
703
704        for (field, value) in [
705            ("start_to_close_timeout", self.start_to_close_timeout),
706            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
707            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
708            ("heartbeat_timeout", self.heartbeat_timeout),
709        ] {
710            if value.is_some_and(|value| value.is_zero()) {
711                return Err(ActivityOptionsError::new(
712                    ActivityOptionsErrorKind::TimeoutNotPositive,
713                    Some(field),
714                    format!("{field} must be positive"),
715                ));
716            }
717        }
718
719        validate_timeout_order(
720            "heartbeat_timeout",
721            self.heartbeat_timeout,
722            "start_to_close_timeout",
723            self.start_to_close_timeout,
724        )?;
725        validate_timeout_order(
726            "start_to_close_timeout",
727            self.start_to_close_timeout,
728            "schedule_to_close_timeout",
729            self.schedule_to_close_timeout,
730        )?;
731        validate_timeout_order(
732            "schedule_to_start_timeout",
733            self.schedule_to_start_timeout,
734            "schedule_to_close_timeout",
735            self.schedule_to_close_timeout,
736        )?;
737
738        Ok(ValidatedActivityOptions {
739            task_queue: self.task_queue.clone(),
740            retry_policy: self
741                .retry_policy
742                .as_ref()
743                .map(validate_activity_retry_policy)
744                .transpose()?,
745            start_to_close_timeout: timeout_seconds(
746                "start_to_close_timeout",
747                self.start_to_close_timeout,
748            )?,
749            schedule_to_start_timeout: timeout_seconds(
750                "schedule_to_start_timeout",
751                self.schedule_to_start_timeout,
752            )?,
753            schedule_to_close_timeout: timeout_seconds(
754                "schedule_to_close_timeout",
755                self.schedule_to_close_timeout,
756            )?,
757            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
758        })
759    }
760}
761
762#[derive(Clone, Debug)]
763struct ValidatedActivityOptions {
764    task_queue: Option<String>,
765    retry_policy: Option<Value>,
766    start_to_close_timeout: Option<u64>,
767    schedule_to_start_timeout: Option<u64>,
768    schedule_to_close_timeout: Option<u64>,
769    heartbeat_timeout: Option<u64>,
770}
771
772fn validate_timeout_order(
773    smaller_name: &'static str,
774    smaller: Option<Duration>,
775    larger_name: &'static str,
776    larger: Option<Duration>,
777) -> std::result::Result<(), ActivityOptionsError> {
778    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
779        return Err(ActivityOptionsError::new(
780            ActivityOptionsErrorKind::TimeoutOrder,
781            Some(smaller_name),
782            format!("{smaller_name} must be <= {larger_name}"),
783        ));
784    }
785    Ok(())
786}
787
788fn timeout_seconds(
789    field: &'static str,
790    value: Option<Duration>,
791) -> std::result::Result<Option<u64>, ActivityOptionsError> {
792    value
793        .map(|value| {
794            activity_protocol_seconds(value).ok_or_else(|| {
795                ActivityOptionsError::new(
796                    ActivityOptionsErrorKind::TimeoutOverflow,
797                    Some(field),
798                    format!("{field} is too large for the worker protocol"),
799                )
800            })
801        })
802        .transpose()
803}
804
805fn duration_seconds_ceil(value: Duration) -> Option<u64> {
806    value
807        .as_secs()
808        .checked_add(u64::from(value.subsec_nanos() > 0))
809}
810
811fn activity_protocol_seconds(value: Duration) -> Option<u64> {
812    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
813}
814
815fn validate_activity_retry_policy(
816    policy: &ActivityRetryPolicy,
817) -> std::result::Result<Value, ActivityOptionsError> {
818    if policy.max_attempts.is_none()
819        && policy.backoff.is_none()
820        && policy.non_retryable_error_types.is_empty()
821    {
822        return Err(ActivityOptionsError::new(
823            ActivityOptionsErrorKind::EmptyRetryPolicy,
824            Some("retry_policy"),
825            "retry_policy must configure at least one field",
826        ));
827    }
828    if policy.max_attempts == Some(0) {
829        return Err(ActivityOptionsError::new(
830            ActivityOptionsErrorKind::InvalidMaxAttempts,
831            Some("retry_policy.max_attempts"),
832            "max_attempts must be >= 1",
833        ));
834    }
835    if policy
836        .non_retryable_error_types
837        .iter()
838        .any(|error_type| error_type.trim().is_empty())
839    {
840        return Err(ActivityOptionsError::new(
841            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
842            Some("retry_policy.non_retryable_error_types"),
843            "non_retryable_error_types must not contain empty values",
844        ));
845    }
846
847    let backoff_seconds = match &policy.backoff {
848        None => None,
849        Some(backoff) => {
850            let max_attempts = policy.max_attempts.ok_or_else(|| {
851                ActivityOptionsError::new(
852                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
853                    Some("retry_policy.backoff"),
854                    "backoff requires max_attempts",
855                )
856            })?;
857            let retry_count = max_attempts.saturating_sub(1) as usize;
858            let intervals = match backoff {
859                ActivityBackoff::Explicit(intervals) => {
860                    if intervals.len() > retry_count {
861                        return Err(ActivityOptionsError::new(
862                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
863                            Some("retry_policy.backoff"),
864                            "backoff interval count must not exceed max_attempts - 1",
865                        ));
866                    }
867                    intervals.clone()
868                }
869                ActivityBackoff::Exponential {
870                    initial_interval,
871                    coefficient,
872                    maximum_interval,
873                } => {
874                    if *coefficient < 1 {
875                        return Err(ActivityOptionsError::new(
876                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
877                            Some("retry_policy.backoff.coefficient"),
878                            "backoff coefficient must be >= 1",
879                        ));
880                    }
881                    if retry_count > 10_000 {
882                        return Err(ActivityOptionsError::new(
883                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
884                            Some("retry_policy.max_attempts"),
885                            "generated backoff supports at most 10000 retry intervals",
886                        ));
887                    }
888                    let mut current = *initial_interval;
889                    let mut intervals = Vec::with_capacity(retry_count);
890                    for _ in 0..retry_count {
891                        let interval = maximum_interval
892                            .map(|maximum| current.min(maximum))
893                            .unwrap_or(current);
894                        intervals.push(interval);
895                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
896                            break;
897                        }
898                        current = current.checked_mul(*coefficient).ok_or_else(|| {
899                            ActivityOptionsError::new(
900                                ActivityOptionsErrorKind::BackoffOverflow,
901                                Some("retry_policy.backoff"),
902                                "generated backoff interval overflowed",
903                            )
904                        })?;
905                    }
906                    intervals
907                }
908            };
909            Some(
910                intervals
911                    .into_iter()
912                    .map(|interval| {
913                        activity_protocol_seconds(interval).ok_or_else(|| {
914                            ActivityOptionsError::new(
915                                ActivityOptionsErrorKind::BackoffOverflow,
916                                Some("retry_policy.backoff"),
917                                "backoff interval is too large for the worker protocol",
918                            )
919                        })
920                    })
921                    .collect::<std::result::Result<Vec<_>, _>>()?,
922            )
923        }
924    };
925
926    let mut encoded = serde_json::Map::new();
927    if let Some(max_attempts) = policy.max_attempts {
928        encoded.insert("max_attempts".to_string(), json!(max_attempts));
929    }
930    if let Some(backoff_seconds) = backoff_seconds {
931        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
932    }
933    if !policy.non_retryable_error_types.is_empty() {
934        let mut canonical_error_types = Vec::new();
935        for error_type in policy
936            .non_retryable_error_types
937            .iter()
938            .map(|error_type| error_type.trim())
939        {
940            if !canonical_error_types.contains(&error_type) {
941                canonical_error_types.push(error_type);
942            }
943        }
944        encoded.insert(
945            "non_retryable_error_types".to_string(),
946            json!(canonical_error_types),
947        );
948    }
949    Ok(Value::Object(encoded))
950}
951
952/// A stable, machine-readable failure raised when workflow code no longer
953/// reconstructs the durable command stream recorded in history.
954#[derive(Clone, Debug, Error)]
955#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
956pub struct ReplayFailure {
957    pub reason: String,
958    pub sequence: Option<u64>,
959    pub expected: Option<String>,
960    pub actual: Option<String>,
961    pub message: String,
962}
963
964impl ReplayFailure {
965    fn new(
966        reason: impl Into<String>,
967        sequence: Option<u64>,
968        expected: Option<String>,
969        actual: Option<String>,
970        message: impl Into<String>,
971    ) -> Self {
972        Self {
973            reason: reason.into(),
974            sequence,
975            expected,
976            actual,
977            message: message.into(),
978        }
979    }
980}
981
982/// A stable, machine-readable workflow query or query-task settlement failure.
983#[derive(Clone, Debug, Error)]
984#[error("query failed ({reason}, HTTP {status}): {message}")]
985pub struct QueryFailure {
986    pub status: u16,
987    pub reason: String,
988    pub message: String,
989    pub body: Value,
990}
991
992/// A stable failure returned when a server rejects an SDK protocol version.
993#[derive(Clone, Debug, Error)]
994#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
995pub struct ProtocolFailure {
996    pub status: u16,
997    pub reason: String,
998    pub message: String,
999    pub supported_version: Option<String>,
1000    pub requested_version: Option<String>,
1001    pub body: Value,
1002}
1003
1004#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1005pub struct PayloadEnvelope {
1006    pub codec: String,
1007    pub blob: String,
1008}
1009
1010impl PayloadEnvelope {
1011    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1012        encode_payload(value, DEFAULT_CODEC)
1013    }
1014
1015    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1016        encode_payload(value, JSON_CODEC)
1017    }
1018
1019    /// Encode an explicit typed value, including the bytes branch that JSON
1020    /// serialization cannot represent.
1021    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1022        encode_avro_value(value)
1023    }
1024}
1025
1026/// Native adapter for the fixed language-neutral Avro Value schema.
1027#[derive(Clone, Debug, PartialEq)]
1028pub enum AvroValue {
1029    Null,
1030    Boolean(bool),
1031    Long(i64),
1032    Double(f64),
1033    Bytes(Vec<u8>),
1034    String(String),
1035    Array(Vec<AvroValue>),
1036    Map(BTreeMap<String, AvroValue>),
1037}
1038
1039impl AvroValue {
1040    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1041        Self::from_serde_value(
1042            serde_value::to_value(value).map_err(|error| {
1043                Error::Codec(format!("could not adapt value for Avro: {error}"))
1044            })?,
1045        )
1046    }
1047
1048    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1049        use serde_value::Value as SerdeValue;
1050
1051        match value {
1052            SerdeValue::Unit => Ok(Self::Null),
1053            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1054            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1055            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1056            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1057            SerdeValue::I64(value) => Ok(Self::Long(value)),
1058            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1059            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1060            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1061            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1062                Error::Codec(
1063                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1064                        .to_string(),
1065                )
1066            }),
1067            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1068            SerdeValue::F64(value) => Self::finite_double(value),
1069            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1070            SerdeValue::String(value) => Ok(Self::String(value)),
1071            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1072            SerdeValue::Option(None) => Ok(Self::Null),
1073            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1074                Self::from_serde_value(*value)
1075            }
1076            SerdeValue::Seq(values) => values
1077                .into_iter()
1078                .map(Self::from_serde_value)
1079                .collect::<Result<Vec<_>>>()
1080                .map(Self::Array),
1081            SerdeValue::Map(values) => values
1082                .into_iter()
1083                .map(|(key, value)| {
1084                    let SerdeValue::String(key) = key else {
1085                        return Err(Error::Codec(
1086                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1087                        ));
1088                    };
1089
1090                    Ok((key, Self::from_serde_value(value)?))
1091                })
1092                .collect::<Result<BTreeMap<_, _>>>()
1093                .map(Self::Map),
1094        }
1095    }
1096
1097    fn finite_double(value: f64) -> Result<Self> {
1098        if !value.is_finite() {
1099            return Err(Error::Codec(
1100                "non_finite_float: Avro Value doubles must be finite".to_string(),
1101            ));
1102        }
1103
1104        Ok(Self::Double(value))
1105    }
1106
1107    fn into_json(self) -> Result<Value> {
1108        match self {
1109            Self::Null => Ok(Value::Null),
1110            Self::Boolean(value) => Ok(Value::Bool(value)),
1111            Self::Long(value) => Ok(Value::Number(value.into())),
1112            Self::Double(value) => serde_json::Number::from_f64(value)
1113                .map(Value::Number)
1114                .ok_or_else(|| {
1115                    Error::Codec(
1116                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1117                    )
1118                }),
1119            Self::Bytes(value) => Ok(json!({
1120                "$type": "bytes",
1121                "base64": BASE64.encode(value),
1122            })),
1123            Self::String(value) => Ok(Value::String(value)),
1124            Self::Array(values) => values
1125                .into_iter()
1126                .map(Self::into_json)
1127                .collect::<Result<Vec<_>>>()
1128                .map(Value::Array),
1129            Self::Map(values) => values
1130                .into_iter()
1131                .map(|(key, value)| Ok((key, value.into_json()?)))
1132                .collect::<Result<serde_json::Map<_, _>>>()
1133                .map(Value::Object),
1134        }
1135    }
1136
1137    fn into_serde_value(self) -> serde_value::Value {
1138        use serde_value::Value as SerdeValue;
1139
1140        match self {
1141            Self::Null => SerdeValue::Unit,
1142            Self::Boolean(value) => SerdeValue::Bool(value),
1143            Self::Long(value) => SerdeValue::I64(value),
1144            Self::Double(value) => SerdeValue::F64(value),
1145            Self::Bytes(value) => SerdeValue::Bytes(value),
1146            Self::String(value) => SerdeValue::String(value),
1147            Self::Array(values) => {
1148                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1149            }
1150            Self::Map(values) => SerdeValue::Map(
1151                values
1152                    .into_iter()
1153                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1154                    .collect(),
1155            ),
1156        }
1157    }
1158
1159    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1160        self.into_serde_value().deserialize_into().map_err(|error| {
1161            Error::Codec(format!(
1162                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1163            ))
1164        })
1165    }
1166}
1167
1168impl Serialize for AvroValue {
1169    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1170    where
1171        S: Serializer,
1172    {
1173        match self {
1174            Self::Null => serializer.serialize_unit(),
1175            Self::Boolean(value) => serializer.serialize_bool(*value),
1176            Self::Long(value) => serializer.serialize_i64(*value),
1177            Self::Double(value) => serializer.serialize_f64(*value),
1178            Self::Bytes(value) => serializer.serialize_bytes(value),
1179            Self::String(value) => serializer.serialize_str(value),
1180            Self::Array(values) => {
1181                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1182                for value in values {
1183                    sequence.serialize_element(value)?;
1184                }
1185                sequence.end()
1186            }
1187            Self::Map(values) => {
1188                let mut map = serializer.serialize_map(Some(values.len()))?;
1189                for (key, value) in values {
1190                    map.serialize_entry(key, value)?;
1191                }
1192                map.end()
1193            }
1194        }
1195    }
1196}
1197
1198pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1199    let datum = avro_value_to_datum(value)?;
1200    let datum = to_avro_datum(avro_value_schema()?, datum)
1201        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1202    let mut bytes = Vec::with_capacity(datum.len() + 10);
1203    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1204    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1205    bytes.extend_from_slice(&datum);
1206    Ok(PayloadEnvelope {
1207        codec: DEFAULT_CODEC.to_string(),
1208        blob: BASE64.encode(bytes),
1209    })
1210}
1211
1212pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1213    if envelope.codec != DEFAULT_CODEC {
1214        return Err(Error::Codec(format!(
1215            "unsupported payload codec {:?}",
1216            envelope.codec
1217        )));
1218    }
1219    decode_avro_value_blob(&envelope.blob)
1220}
1221
1222pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1223    let blob = match codec {
1224        JSON_CODEC => serde_json::to_string(value)?,
1225        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1226        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1227    };
1228
1229    Ok(PayloadEnvelope {
1230        codec: codec.to_string(),
1231        blob,
1232    })
1233}
1234
1235pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1236    match envelope.codec.as_str() {
1237        JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1238        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1239        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1240    }
1241}
1242
1243#[cfg(test)]
1244fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1245    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1246}
1247
1248fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1249    if value.is_null() {
1250        return Ok(Value::Null);
1251    }
1252
1253    if let Some(object) = value.as_object() {
1254        if let (Some(codec), Some(blob)) = (
1255            object.get("codec").and_then(Value::as_str),
1256            object.get("blob").and_then(Value::as_str),
1257        ) {
1258            return decode_blob(blob, codec);
1259        }
1260    }
1261
1262    if let Some(blob) = value.as_str() {
1263        return decode_blob(blob, fallback_codec);
1264    }
1265
1266    Ok(value.clone())
1267}
1268
1269fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1270    let envelope = match codec {
1271        DEFAULT_CODEC => encode_avro_value(value)?,
1272        JSON_CODEC => PayloadEnvelope {
1273            codec: JSON_CODEC.to_string(),
1274            blob: serde_json::to_string(&value.clone().into_json()?)?,
1275        },
1276        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1277    };
1278    Ok(serde_json::to_value(envelope)?)
1279}
1280
1281fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1282    if value.is_null() {
1283        return Ok(AvroValue::Null);
1284    }
1285
1286    if let Some(object) = value.as_object() {
1287        if let (Some(codec), Some(blob)) = (
1288            object.get("codec").and_then(Value::as_str),
1289            object.get("blob").and_then(Value::as_str),
1290        ) {
1291            return match codec {
1292                DEFAULT_CODEC => decode_avro_value_blob(blob),
1293                JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1294                other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1295            };
1296        }
1297    }
1298
1299    if let Some(blob) = value.as_str() {
1300        return match fallback_codec {
1301            DEFAULT_CODEC => decode_avro_value_blob(blob),
1302            JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1303            other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1304        };
1305    }
1306
1307    AvroValue::from_serialize(value)
1308}
1309
1310fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1311    match value {
1312        AvroValue::Null => AvroValue::Array(Vec::new()),
1313        AvroValue::Array(_) => value,
1314        other => AvroValue::Array(vec![other]),
1315    }
1316}
1317
1318fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1319    match codec {
1320        JSON_CODEC => Ok(serde_json::from_str(blob)?),
1321        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1322        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1323    }
1324}
1325
1326fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1327    let bytes = BASE64.decode(blob).map_err(|err| {
1328        Error::Codec(format!(
1329            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1330        ))
1331    })?;
1332
1333    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1334        return Err(Error::Codec(
1335            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1336        ));
1337    }
1338
1339    let fingerprint: [u8; 8] = bytes[2..10]
1340        .try_into()
1341        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1342    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1343        return Err(Error::Codec(format!(
1344            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1345            fingerprint
1346                .iter()
1347                .map(|byte| format!("{byte:02x}"))
1348                .collect::<String>()
1349        )));
1350    }
1351
1352    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1353    // The current fingerprint selects the current immutable schema, so reader
1354    // resolution would only re-walk the same recursive union. Future retained
1355    // writer fingerprints supply a distinct reader schema in this branch.
1356    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1357    if datum_reader.truncated {
1358        return Err(Error::Codec(
1359            "invalid_payload_framing: truncated Avro Value datum".to_string(),
1360        ));
1361    }
1362    let datum = datum.map_err(|err| {
1363        Error::Codec(format!(
1364            "invalid_payload_framing: malformed Avro Value datum: {err}"
1365        ))
1366    })?;
1367    if datum_reader.remaining() != 0 {
1368        return Err(Error::Codec(format!(
1369            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1370            datum_reader.remaining()
1371        )));
1372    }
1373    avro_value_from_datum(datum)
1374}
1375
1376struct StrictAvroDatumReader<'a> {
1377    bytes: &'a [u8],
1378    offset: usize,
1379    truncated: bool,
1380}
1381
1382impl<'a> StrictAvroDatumReader<'a> {
1383    fn new(bytes: &'a [u8]) -> Self {
1384        Self {
1385            bytes,
1386            offset: 0,
1387            truncated: false,
1388        }
1389    }
1390
1391    fn remaining(&self) -> usize {
1392        self.bytes.len() - self.offset
1393    }
1394}
1395
1396impl Read for StrictAvroDatumReader<'_> {
1397    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1398        let count = buffer.len().min(self.remaining());
1399        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1400        self.offset += count;
1401        if count < buffer.len() {
1402            self.truncated = true;
1403        }
1404
1405        Ok(count)
1406    }
1407}
1408
1409fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1410    let branch = match value {
1411        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1412        AvroValue::Boolean(value) => AvroDatum::Union(
1413            1,
1414            Box::new(AvroDatum::Record(vec![(
1415                "boolean".to_string(),
1416                AvroDatum::Boolean(*value),
1417            )])),
1418        ),
1419        AvroValue::Long(value) => AvroDatum::Union(
1420            2,
1421            Box::new(AvroDatum::Record(vec![(
1422                "long".to_string(),
1423                AvroDatum::Long(*value),
1424            )])),
1425        ),
1426        AvroValue::Double(value) => {
1427            if !value.is_finite() {
1428                return Err(Error::Codec(
1429                    "non_finite_float: Avro Value doubles must be finite".to_string(),
1430                ));
1431            }
1432            AvroDatum::Union(
1433                3,
1434                Box::new(AvroDatum::Record(vec![(
1435                    "double".to_string(),
1436                    AvroDatum::Double(*value),
1437                )])),
1438            )
1439        }
1440        AvroValue::Bytes(value) => AvroDatum::Union(
1441            4,
1442            Box::new(AvroDatum::Record(vec![(
1443                "bytes".to_string(),
1444                AvroDatum::Bytes(value.clone()),
1445            )])),
1446        ),
1447        AvroValue::String(value) => AvroDatum::Union(
1448            5,
1449            Box::new(AvroDatum::Record(vec![(
1450                "string".to_string(),
1451                AvroDatum::String(value.clone()),
1452            )])),
1453        ),
1454        AvroValue::Array(values) => AvroDatum::Union(
1455            6,
1456            Box::new(AvroDatum::Record(vec![(
1457                "items".to_string(),
1458                AvroDatum::Array(
1459                    values
1460                        .iter()
1461                        .map(avro_value_to_datum)
1462                        .collect::<Result<Vec<_>>>()?,
1463                ),
1464            )])),
1465        ),
1466        AvroValue::Map(values) => AvroDatum::Union(
1467            7,
1468            Box::new(AvroDatum::Record(vec![(
1469                "entries".to_string(),
1470                AvroDatum::Map(
1471                    values
1472                        .iter()
1473                        .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1474                        .collect::<Result<HashMap<_, _>>>()?,
1475                ),
1476            )])),
1477        ),
1478    };
1479    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1480}
1481
1482fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1483    let AvroDatum::Record(mut outer) = datum else {
1484        return Err(Error::Codec(
1485            "invalid_payload_framing: datum is not a Value record".to_string(),
1486        ));
1487    };
1488    let (_, branch) = outer
1489        .pop()
1490        .filter(|(name, _)| name == "value")
1491        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1492    let AvroDatum::Union(_, branch) = branch else {
1493        return Err(Error::Codec(
1494            "invalid_payload_framing: invalid Value union".to_string(),
1495        ));
1496    };
1497    match *branch {
1498        AvroDatum::Null => Ok(AvroValue::Null),
1499        AvroDatum::Record(mut fields) => {
1500            let (name, value) = fields.pop().ok_or_else(|| {
1501                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1502            })?;
1503            match (name.as_str(), value) {
1504                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1505                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1506                ("double", AvroDatum::Double(value)) if value.is_finite() => {
1507                    Ok(AvroValue::Double(value))
1508                }
1509                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1510                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1511                ("items", AvroDatum::Array(values)) => values
1512                    .into_iter()
1513                    .map(avro_value_from_datum)
1514                    .collect::<Result<Vec<_>>>()
1515                    .map(AvroValue::Array),
1516                ("entries", AvroDatum::Map(values)) => values
1517                    .into_iter()
1518                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1519                    .collect::<Result<BTreeMap<_, _>>>()
1520                    .map(AvroValue::Map),
1521                _ => Err(Error::Codec(
1522                    "invalid_payload_framing: unknown Value branch".to_string(),
1523                )),
1524            }
1525        }
1526        _ => Err(Error::Codec(
1527            "invalid_payload_framing: invalid Value branch".to_string(),
1528        )),
1529    }
1530}
1531
1532fn avro_value_schema() -> Result<&'static Schema> {
1533    match AVRO_VALUE_SCHEMA.get_or_init(|| {
1534        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1535            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1536    }) {
1537        Ok(schema) => Ok(schema),
1538        Err(message) => Err(Error::Codec(message.clone())),
1539    }
1540}
1541
1542#[derive(Clone, Debug)]
1543pub struct Client {
1544    http: reqwest::Client,
1545    base_url: String,
1546    token: Option<String>,
1547    control_token: Option<String>,
1548    worker_token: Option<String>,
1549    namespace: String,
1550}
1551
1552impl Client {
1553    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1554        Self::builder(base_url).build()
1555    }
1556
1557    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1558        ClientBuilder {
1559            base_url: base_url.into(),
1560            token: None,
1561            control_token: None,
1562            worker_token: None,
1563            namespace: "default".to_string(),
1564            timeout: Duration::from_secs(60),
1565        }
1566    }
1567
1568    pub async fn health(&self) -> Result<Value> {
1569        self.request_json(
1570            reqwest::Method::GET,
1571            "/health",
1572            RequestProtocol::ControlPlane,
1573            Option::<&Value>::None,
1574        )
1575        .await
1576    }
1577
1578    pub async fn cluster_info(&self) -> Result<Value> {
1579        self.request_json(
1580            reqwest::Method::GET,
1581            "/cluster/info",
1582            RequestProtocol::ControlPlane,
1583            Option::<&Value>::None,
1584        )
1585        .await
1586    }
1587
1588    pub async fn start_workflow<T: Serialize>(
1589        &self,
1590        workflow_type: &str,
1591        task_queue: &str,
1592        workflow_id: &str,
1593        input: T,
1594    ) -> Result<WorkflowHandle> {
1595        self.start_workflow_with_options(
1596            workflow_type,
1597            task_queue,
1598            workflow_id,
1599            WorkflowStartOptions::default(),
1600            input,
1601        )
1602        .await
1603    }
1604
1605    /// Start a workflow with explicit server-enforced execution and run
1606    /// deadlines.
1607    pub async fn start_workflow_with_options<T: Serialize>(
1608        &self,
1609        workflow_type: &str,
1610        task_queue: &str,
1611        workflow_id: &str,
1612        options: WorkflowStartOptions,
1613        input: T,
1614    ) -> Result<WorkflowHandle> {
1615        options.validate()?;
1616        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1617        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1618        let body = json!({
1619            "workflow_id": workflow_id,
1620            "workflow_type": workflow_type,
1621            "task_queue": task_queue,
1622            "input": input_envelope,
1623            "execution_timeout_seconds": options.execution_timeout_seconds,
1624            "run_timeout_seconds": options.run_timeout_seconds
1625        });
1626
1627        let data: Value = self
1628            .request_json(
1629                reqwest::Method::POST,
1630                "/workflows",
1631                RequestProtocol::ControlPlane,
1632                Some(&body),
1633            )
1634            .await?;
1635
1636        Ok(WorkflowHandle {
1637            client: self.clone(),
1638            workflow_id: data
1639                .get("workflow_id")
1640                .and_then(Value::as_str)
1641                .unwrap_or(workflow_id)
1642                .to_string(),
1643            run_id: data
1644                .get("run_id")
1645                .and_then(Value::as_str)
1646                .map(str::to_string),
1647            workflow_type: data
1648                .get("workflow_type")
1649                .and_then(Value::as_str)
1650                .unwrap_or(workflow_type)
1651                .to_string(),
1652        })
1653    }
1654
1655    pub async fn signal_workflow<T: Serialize>(
1656        &self,
1657        workflow_id: &str,
1658        signal_name: &str,
1659        input: T,
1660    ) -> Result<Value> {
1661        self.signal_workflow_target(workflow_id, None, signal_name, input)
1662            .await
1663    }
1664
1665    /// Signal only if `run_id` is still the current run for this instance.
1666    pub async fn signal_workflow_run<T: Serialize>(
1667        &self,
1668        workflow_id: &str,
1669        run_id: &str,
1670        signal_name: &str,
1671        input: T,
1672    ) -> Result<Value> {
1673        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1674            .await
1675    }
1676
1677    async fn signal_workflow_target<T: Serialize>(
1678        &self,
1679        workflow_id: &str,
1680        run_id: Option<&str>,
1681        signal_name: &str,
1682        input: T,
1683    ) -> Result<Value> {
1684        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1685        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1686        let body = json!({
1687            "input": input_envelope
1688        });
1689        let path = match run_id {
1690            Some(run_id) => {
1691                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1692            }
1693            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1694        };
1695        self.request_json(
1696            reqwest::Method::POST,
1697            &path,
1698            RequestProtocol::ControlPlane,
1699            Some(&body),
1700        )
1701        .await
1702    }
1703
1704    /// Request cooperative cancellation of the current run for an instance.
1705    pub async fn cancel_workflow(
1706        &self,
1707        workflow_id: &str,
1708        options: WorkflowCommandOptions,
1709    ) -> Result<WorkflowCommandResult> {
1710        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1711            .await
1712    }
1713
1714    /// Request cooperative cancellation only if `run_id` is still current.
1715    pub async fn cancel_workflow_run(
1716        &self,
1717        workflow_id: &str,
1718        run_id: &str,
1719        options: WorkflowCommandOptions,
1720    ) -> Result<WorkflowCommandResult> {
1721        self.workflow_command(
1722            workflow_id,
1723            Some(run_id),
1724            WorkflowCommandKind::Cancel,
1725            options,
1726        )
1727        .await
1728    }
1729
1730    /// Forcefully terminate the current run for an instance.
1731    pub async fn terminate_workflow(
1732        &self,
1733        workflow_id: &str,
1734        options: WorkflowCommandOptions,
1735    ) -> Result<WorkflowCommandResult> {
1736        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1737            .await
1738    }
1739
1740    /// Forcefully terminate only if `run_id` is still current.
1741    pub async fn terminate_workflow_run(
1742        &self,
1743        workflow_id: &str,
1744        run_id: &str,
1745        options: WorkflowCommandOptions,
1746    ) -> Result<WorkflowCommandResult> {
1747        self.workflow_command(
1748            workflow_id,
1749            Some(run_id),
1750            WorkflowCommandKind::Terminate,
1751            options,
1752        )
1753        .await
1754    }
1755
1756    async fn workflow_command(
1757        &self,
1758        workflow_id: &str,
1759        run_id: Option<&str>,
1760        command: WorkflowCommandKind,
1761        options: WorkflowCommandOptions,
1762    ) -> Result<WorkflowCommandResult> {
1763        let path = match run_id {
1764            Some(run_id) => format!(
1765                "/workflows/{workflow_id}/runs/{run_id}/{}",
1766                command.as_str()
1767            ),
1768            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1769        };
1770        let data = match self
1771            .request_json(
1772                reqwest::Method::POST,
1773                &path,
1774                RequestProtocol::ControlPlane,
1775                Some(&options),
1776            )
1777            .await
1778        {
1779            Ok(data) => data,
1780            Err(Error::Http { status, body }) => {
1781                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1782                    command,
1783                    status,
1784                    body,
1785                    workflow_id,
1786                    run_id,
1787                )));
1788            }
1789            Err(error) => return Err(error),
1790        };
1791
1792        Ok(workflow_command_result(command, data, workflow_id, run_id))
1793    }
1794
1795    /// Execute a named, read-only query against a running or completed workflow.
1796    ///
1797    /// Arguments and results use the platform payload envelope. Server and
1798    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1799    /// reason, HTTP status, and original response body.
1800    pub async fn query_workflow<T: Serialize>(
1801        &self,
1802        workflow_id: &str,
1803        query_name: &str,
1804        input: T,
1805    ) -> Result<Value> {
1806        self.query_workflow_target(workflow_id, None, query_name, input)
1807            .await
1808    }
1809
1810    /// Query only if `run_id` is still current, preventing accidental retargeting.
1811    pub async fn query_workflow_run<T: Serialize>(
1812        &self,
1813        workflow_id: &str,
1814        run_id: &str,
1815        query_name: &str,
1816        input: T,
1817    ) -> Result<Value> {
1818        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1819            .await
1820    }
1821
1822    /// Query a workflow and return the lossless fixed Avro Value result.
1823    pub async fn query_workflow_avro_value<T: Serialize>(
1824        &self,
1825        workflow_id: &str,
1826        query_name: &str,
1827        input: T,
1828    ) -> Result<AvroValue> {
1829        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1830            .await
1831    }
1832
1833    /// Query a selected run and return the lossless fixed Avro Value result.
1834    pub async fn query_workflow_run_avro_value<T: Serialize>(
1835        &self,
1836        workflow_id: &str,
1837        run_id: &str,
1838        query_name: &str,
1839        input: T,
1840    ) -> Result<AvroValue> {
1841        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1842            .await
1843    }
1844
1845    async fn query_workflow_avro_value_target<T: Serialize>(
1846        &self,
1847        workflow_id: &str,
1848        run_id: Option<&str>,
1849        query_name: &str,
1850        input: T,
1851    ) -> Result<AvroValue> {
1852        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1853        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1854        let path = match run_id {
1855            Some(run_id) => {
1856                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1857            }
1858            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1859        };
1860        let response: Value = match self
1861            .request_json(
1862                reqwest::Method::POST,
1863                &path,
1864                RequestProtocol::ControlPlane,
1865                Some(&body),
1866            )
1867            .await
1868        {
1869            Ok(response) => response,
1870            Err(Error::Http { status, body }) => {
1871                return Err(Error::QueryFailed(query_failure(status, body)));
1872            }
1873            Err(error) => return Err(error),
1874        };
1875
1876        let envelope = response
1877            .get("result_envelope")
1878            .filter(|envelope| !envelope.is_null())
1879            .ok_or_else(|| {
1880                Error::Codec(
1881                    "missing_payload_envelope: typed query result requires result_envelope"
1882                        .to_string(),
1883                )
1884            })?;
1885        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1886    }
1887
1888    async fn query_workflow_target<T: Serialize>(
1889        &self,
1890        workflow_id: &str,
1891        run_id: Option<&str>,
1892        query_name: &str,
1893        input: T,
1894    ) -> Result<Value> {
1895        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1896        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1897        let body = json!({
1898            "input": input_envelope
1899        });
1900        let path = match run_id {
1901            Some(run_id) => {
1902                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1903            }
1904            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1905        };
1906        let response: Value = match self
1907            .request_json(
1908                reqwest::Method::POST,
1909                &path,
1910                RequestProtocol::ControlPlane,
1911                Some(&body),
1912            )
1913            .await
1914        {
1915            Ok(response) => response,
1916            Err(Error::Http { status, body }) => {
1917                return Err(Error::QueryFailed(query_failure(status, body)));
1918            }
1919            Err(error) => return Err(error),
1920        };
1921
1922        if let Some(envelope) = response
1923            .get("result_envelope")
1924            .filter(|envelope| !envelope.is_null())
1925        {
1926            return decode_wire_value(envelope, DEFAULT_CODEC);
1927        }
1928
1929        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1930    }
1931
1932    /// Send a synchronous update using fixed Avro Value arguments.
1933    pub async fn update_workflow<T: Serialize>(
1934        &self,
1935        workflow_id: &str,
1936        update_name: &str,
1937        input: T,
1938        request_id: Option<&str>,
1939    ) -> Result<Value> {
1940        let response = self
1941            .update_workflow_response(workflow_id, update_name, input, request_id)
1942            .await?;
1943        if let Some(envelope) = response
1944            .get("result_envelope")
1945            .filter(|envelope| !envelope.is_null())
1946        {
1947            return decode_wire_value(envelope, DEFAULT_CODEC);
1948        }
1949        Ok(response.get("result").cloned().unwrap_or(response))
1950    }
1951
1952    /// Send a synchronous update and retain a bytes-capable Avro result.
1953    pub async fn update_workflow_avro_value<T: Serialize>(
1954        &self,
1955        workflow_id: &str,
1956        update_name: &str,
1957        input: T,
1958        request_id: Option<&str>,
1959    ) -> Result<AvroValue> {
1960        let response = self
1961            .update_workflow_response(workflow_id, update_name, input, request_id)
1962            .await?;
1963        let envelope = response
1964            .get("result_envelope")
1965            .filter(|envelope| !envelope.is_null())
1966            .ok_or_else(|| {
1967                Error::Codec(
1968                    "missing_payload_envelope: typed update result requires result_envelope"
1969                        .to_string(),
1970                )
1971            })?;
1972        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1973    }
1974
1975    async fn update_workflow_response<T: Serialize>(
1976        &self,
1977        workflow_id: &str,
1978        update_name: &str,
1979        input: T,
1980        request_id: Option<&str>,
1981    ) -> Result<Value> {
1982        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1983        let mut body = json!({
1984            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1985            "wait_for": "completed",
1986        });
1987        if let Some(request_id) = request_id {
1988            body["request_id"] = json!(request_id);
1989        }
1990        self.request_json(
1991            reqwest::Method::POST,
1992            &format!("/workflows/{workflow_id}/update/{update_name}"),
1993            RequestProtocol::ControlPlane,
1994            Some(&body),
1995        )
1996        .await
1997    }
1998
1999    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2000        let path = format!("/workflows/{workflow_id}");
2001        let mut data: WorkflowDescription = self
2002            .request_json(
2003                reqwest::Method::GET,
2004                &path,
2005                RequestProtocol::ControlPlane,
2006                Option::<&Value>::None,
2007            )
2008            .await?;
2009        data.decode_payloads()?;
2010        Ok(data)
2011    }
2012
2013    /// Describe one selected run, including historical terminal runs.
2014    pub async fn describe_workflow_run(
2015        &self,
2016        workflow_id: &str,
2017        run_id: &str,
2018    ) -> Result<WorkflowDescription> {
2019        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2020        let mut data: WorkflowDescription = self
2021            .request_json(
2022                reqwest::Method::GET,
2023                &path,
2024                RequestProtocol::ControlPlane,
2025                Option::<&Value>::None,
2026            )
2027            .await?;
2028        data.decode_payloads()?;
2029        Ok(data)
2030    }
2031
2032    pub async fn register_worker(
2033        &self,
2034        worker_id: &str,
2035        task_queue: &str,
2036        supported_workflow_types: Vec<String>,
2037        supported_activity_types: Vec<String>,
2038        max_concurrent_workflow_tasks: usize,
2039        max_concurrent_activity_tasks: usize,
2040    ) -> Result<RegisterWorkerResponse> {
2041        self.register_worker_with_capabilities(
2042            worker_id,
2043            task_queue,
2044            supported_workflow_types,
2045            supported_activity_types,
2046            max_concurrent_workflow_tasks,
2047            max_concurrent_activity_tasks,
2048            Vec::new(),
2049        )
2050        .await
2051    }
2052
2053    /// Register a worker and explicitly advertise additive worker capabilities.
2054    pub async fn register_worker_with_capabilities(
2055        &self,
2056        worker_id: &str,
2057        task_queue: &str,
2058        supported_workflow_types: Vec<String>,
2059        supported_activity_types: Vec<String>,
2060        max_concurrent_workflow_tasks: usize,
2061        max_concurrent_activity_tasks: usize,
2062        capabilities: Vec<String>,
2063    ) -> Result<RegisterWorkerResponse> {
2064        self.register_worker_with_command_contracts(
2065            worker_id,
2066            task_queue,
2067            supported_workflow_types,
2068            supported_activity_types,
2069            max_concurrent_workflow_tasks,
2070            max_concurrent_activity_tasks,
2071            capabilities,
2072            Value::Object(serde_json::Map::new()),
2073        )
2074        .await
2075    }
2076
2077    /// Register a worker and advertise its named query and update handlers.
2078    #[allow(clippy::too_many_arguments)]
2079    pub async fn register_worker_with_command_contracts(
2080        &self,
2081        worker_id: &str,
2082        task_queue: &str,
2083        supported_workflow_types: Vec<String>,
2084        supported_activity_types: Vec<String>,
2085        max_concurrent_workflow_tasks: usize,
2086        max_concurrent_activity_tasks: usize,
2087        capabilities: Vec<String>,
2088        workflow_command_contracts: Value,
2089    ) -> Result<RegisterWorkerResponse> {
2090        let mut body = json!({
2091            "worker_id": worker_id,
2092            "task_queue": task_queue,
2093            "runtime": "rust",
2094            "sdk_version": SDK_VERSION,
2095            "supported_workflow_types": supported_workflow_types,
2096            "supported_activity_types": supported_activity_types,
2097            "capabilities": capabilities,
2098            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2099            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2100        });
2101        if workflow_command_contracts
2102            .as_object()
2103            .is_some_and(|contracts| !contracts.is_empty())
2104        {
2105            body["workflow_command_contracts"] = workflow_command_contracts;
2106        }
2107
2108        self.request_json(
2109            reqwest::Method::POST,
2110            "/worker/register",
2111            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2112            Some(&body),
2113        )
2114        .await
2115    }
2116
2117    /// Long-poll for an ephemeral, read-only workflow query task.
2118    pub async fn poll_query_task(
2119        &self,
2120        worker_id: &str,
2121        task_queue: &str,
2122        timeout: Duration,
2123    ) -> Result<Option<QueryTask>> {
2124        Ok(self
2125            .poll_query_task_response(worker_id, task_queue, timeout)
2126            .await?
2127            .task)
2128    }
2129
2130    /// Poll a query task while preserving server stop and drain metadata.
2131    pub async fn poll_query_task_response(
2132        &self,
2133        worker_id: &str,
2134        task_queue: &str,
2135        timeout: Duration,
2136    ) -> Result<PollQueryTaskResponse> {
2137        let poll_request_id = unique_request_id("rust-query-poll");
2138        self.poll_query_task_response_with_request_id(
2139            worker_id,
2140            task_queue,
2141            timeout,
2142            &poll_request_id,
2143            1,
2144        )
2145        .await
2146    }
2147
2148    async fn poll_query_task_response_with_request_id(
2149        &self,
2150        worker_id: &str,
2151        task_queue: &str,
2152        timeout: Duration,
2153        poll_request_id: &str,
2154        transport_retries: usize,
2155    ) -> Result<PollQueryTaskResponse> {
2156        let timeout_seconds = long_poll_timeout_seconds(timeout);
2157        let body = json!({
2158            "worker_id": worker_id,
2159            "task_queue": task_queue,
2160            "poll_request_id": poll_request_id,
2161            "timeout_seconds": timeout_seconds,
2162        });
2163        self.poll_request_json(
2164            "/worker/query-tasks/poll",
2165            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2166            &body,
2167            timeout + Duration::from_secs(5),
2168            transport_retries,
2169        )
2170        .await
2171    }
2172
2173    /// Complete a query task without appending workflow history.
2174    pub async fn complete_query_task<T: Serialize>(
2175        &self,
2176        query_task_id: &str,
2177        lease_owner: &str,
2178        query_task_attempt: u64,
2179        result: T,
2180        codec: &str,
2181    ) -> Result<Value> {
2182        let typed_result = AvroValue::from_serialize(&result)?;
2183        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2184        self.complete_query_task_with_envelope(
2185            query_task_id,
2186            lease_owner,
2187            query_task_attempt,
2188            typed_result.into_json()?,
2189            result_envelope,
2190        )
2191        .await
2192    }
2193
2194    async fn complete_query_task_with_envelope(
2195        &self,
2196        query_task_id: &str,
2197        lease_owner: &str,
2198        query_task_attempt: u64,
2199        result: Value,
2200        result_envelope: Value,
2201    ) -> Result<Value> {
2202        let body = json!({
2203            "lease_owner": lease_owner,
2204            "query_task_attempt": query_task_attempt,
2205            "result": result,
2206            "result_envelope": result_envelope,
2207        });
2208        let path = format!("/worker/query-tasks/{query_task_id}/complete");
2209        let response = self
2210            .request_json(
2211                reqwest::Method::POST,
2212                &path,
2213                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2214                Some(&body),
2215            )
2216            .await;
2217        query_task_response(response)
2218    }
2219
2220    /// Report a stable machine-readable query-task failure.
2221    pub async fn fail_query_task(
2222        &self,
2223        query_task_id: &str,
2224        lease_owner: &str,
2225        query_task_attempt: u64,
2226        message: impl Into<String>,
2227        reason: impl Into<String>,
2228        failure_type: impl Into<String>,
2229    ) -> Result<Value> {
2230        let body = json!({
2231            "lease_owner": lease_owner,
2232            "query_task_attempt": query_task_attempt,
2233            "failure": {
2234                "message": message.into(),
2235                "reason": reason.into(),
2236                "type": failure_type.into(),
2237            }
2238        });
2239        let path = format!("/worker/query-tasks/{query_task_id}/fail");
2240        let response = self
2241            .request_json(
2242                reqwest::Method::POST,
2243                &path,
2244                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2245                Some(&body),
2246            )
2247            .await;
2248        query_task_response(response)
2249    }
2250
2251    pub async fn heartbeat_worker(
2252        &self,
2253        worker_id: &str,
2254        workflow_available: usize,
2255        activity_available: usize,
2256    ) -> Result<Value> {
2257        let body = json!({
2258            "worker_id": worker_id,
2259            "task_slots": {
2260                "workflow_available": workflow_available,
2261                "activity_available": activity_available
2262            },
2263            "process_metrics": {
2264                "process_id": std::process::id(),
2265                "process_uptime_seconds": 0
2266            }
2267        });
2268
2269        self.request_json(
2270            reqwest::Method::POST,
2271            "/worker/heartbeat",
2272            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2273            Some(&body),
2274        )
2275        .await
2276    }
2277
2278    pub async fn poll_workflow_task(
2279        &self,
2280        worker_id: &str,
2281        task_queue: &str,
2282        timeout: Duration,
2283    ) -> Result<Option<WorkflowTask>> {
2284        Ok(self
2285            .poll_workflow_task_response(worker_id, task_queue, timeout)
2286            .await?
2287            .task)
2288    }
2289
2290    pub async fn poll_workflow_task_response(
2291        &self,
2292        worker_id: &str,
2293        task_queue: &str,
2294        timeout: Duration,
2295    ) -> Result<PollWorkflowTaskResponse> {
2296        let poll_request_id = unique_request_id("rust-workflow-poll");
2297        self.poll_workflow_task_response_with_request_id(
2298            worker_id,
2299            task_queue,
2300            timeout,
2301            &poll_request_id,
2302            1,
2303        )
2304        .await
2305    }
2306
2307    async fn poll_workflow_task_response_with_request_id(
2308        &self,
2309        worker_id: &str,
2310        task_queue: &str,
2311        timeout: Duration,
2312        poll_request_id: &str,
2313        transport_retries: usize,
2314    ) -> Result<PollWorkflowTaskResponse> {
2315        let body = json!({
2316            "worker_id": worker_id,
2317            "task_queue": task_queue,
2318            "poll_request_id": poll_request_id,
2319            "timeout_seconds": long_poll_timeout_seconds(timeout),
2320        });
2321        let mut data: PollWorkflowTaskResponse = self
2322            .poll_request_json(
2323                "/worker/workflow-tasks/poll",
2324                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2325                &body,
2326                timeout + Duration::from_secs(5),
2327                transport_retries,
2328            )
2329            .await?;
2330
2331        if let Some(task) = data.task.as_mut() {
2332            self.fetch_remaining_workflow_history(worker_id, task)
2333                .await?;
2334        }
2335
2336        Ok(data)
2337    }
2338
2339    async fn fetch_remaining_workflow_history(
2340        &self,
2341        worker_id: &str,
2342        task: &mut WorkflowTask,
2343    ) -> Result<()> {
2344        let mut next_token = task.next_history_page_token.clone();
2345
2346        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2347            let lease_owner = task
2348                .lease_owner
2349                .clone()
2350                .unwrap_or_else(|| worker_id.to_string());
2351            let page = self
2352                .workflow_task_history_page(
2353                    &task.task_id,
2354                    &lease_owner,
2355                    task.workflow_task_attempt,
2356                    &token,
2357                )
2358                .await?;
2359
2360            task.append_history_page(page);
2361
2362            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2363                return Err(Error::Codec(
2364                    "workflow history pagination returned the same page token".to_string(),
2365                ));
2366            }
2367
2368            next_token = task.next_history_page_token.clone();
2369        }
2370
2371        Ok(())
2372    }
2373
2374    async fn workflow_task_history_page(
2375        &self,
2376        task_id: &str,
2377        lease_owner: &str,
2378        workflow_task_attempt: u64,
2379        next_history_page_token: &str,
2380    ) -> Result<WorkflowTaskHistoryPage> {
2381        let body = json!({
2382            "lease_owner": lease_owner,
2383            "workflow_task_attempt": workflow_task_attempt,
2384            "next_history_page_token": next_history_page_token
2385        });
2386        let path = format!("/worker/workflow-tasks/{task_id}/history");
2387
2388        self.request_json(
2389            reqwest::Method::POST,
2390            &path,
2391            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2392            Some(&body),
2393        )
2394        .await
2395    }
2396
2397    pub async fn complete_workflow_task(
2398        &self,
2399        task_id: &str,
2400        lease_owner: &str,
2401        workflow_task_attempt: u64,
2402        commands: Vec<Value>,
2403    ) -> Result<Value> {
2404        let body = json!({
2405            "lease_owner": lease_owner,
2406            "workflow_task_attempt": workflow_task_attempt,
2407            "commands": commands
2408        });
2409        let path = format!("/worker/workflow-tasks/{task_id}/complete");
2410        self.request_json(
2411            reqwest::Method::POST,
2412            &path,
2413            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2414            Some(&body),
2415        )
2416        .await
2417    }
2418
2419    pub async fn fail_workflow_task(
2420        &self,
2421        task_id: &str,
2422        lease_owner: &str,
2423        workflow_task_attempt: u64,
2424        message: impl Into<String>,
2425    ) -> Result<Value> {
2426        self.fail_workflow_task_with_type(
2427            task_id,
2428            lease_owner,
2429            workflow_task_attempt,
2430            message,
2431            "RustWorkflowTaskFailure",
2432        )
2433        .await
2434    }
2435
2436    async fn fail_workflow_task_with_type(
2437        &self,
2438        task_id: &str,
2439        lease_owner: &str,
2440        workflow_task_attempt: u64,
2441        message: impl Into<String>,
2442        failure_type: &str,
2443    ) -> Result<Value> {
2444        let body = json!({
2445            "lease_owner": lease_owner,
2446            "workflow_task_attempt": workflow_task_attempt,
2447            "failure": {
2448                "message": message.into(),
2449                "type": failure_type
2450            }
2451        });
2452        let path = format!("/worker/workflow-tasks/{task_id}/fail");
2453        self.request_json(
2454            reqwest::Method::POST,
2455            &path,
2456            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2457            Some(&body),
2458        )
2459        .await
2460    }
2461
2462    pub async fn poll_activity_task(
2463        &self,
2464        worker_id: &str,
2465        task_queue: &str,
2466        timeout: Duration,
2467    ) -> Result<Option<ActivityTask>> {
2468        Ok(self
2469            .poll_activity_task_response(worker_id, task_queue, timeout)
2470            .await?
2471            .task)
2472    }
2473
2474    /// Poll an activity task while preserving server stop and drain metadata.
2475    pub async fn poll_activity_task_response(
2476        &self,
2477        worker_id: &str,
2478        task_queue: &str,
2479        timeout: Duration,
2480    ) -> Result<PollActivityTaskResponse> {
2481        let poll_request_id = unique_request_id("rust-activity-poll");
2482        self.poll_activity_task_response_with_request_id(
2483            worker_id,
2484            task_queue,
2485            timeout,
2486            &poll_request_id,
2487            1,
2488        )
2489        .await
2490    }
2491
2492    async fn poll_activity_task_response_with_request_id(
2493        &self,
2494        worker_id: &str,
2495        task_queue: &str,
2496        timeout: Duration,
2497        poll_request_id: &str,
2498        transport_retries: usize,
2499    ) -> Result<PollActivityTaskResponse> {
2500        let body = json!({
2501            "worker_id": worker_id,
2502            "task_queue": task_queue,
2503            "poll_request_id": poll_request_id,
2504            "timeout_seconds": long_poll_timeout_seconds(timeout),
2505        });
2506        let data: PollActivityTaskResponse = self
2507            .poll_request_json(
2508                "/worker/activity-tasks/poll",
2509                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2510                &body,
2511                timeout + Duration::from_secs(5),
2512                transport_retries,
2513            )
2514            .await?;
2515        Ok(data)
2516    }
2517
2518    pub async fn complete_activity_task<T: Serialize>(
2519        &self,
2520        task_id: &str,
2521        activity_attempt_id: &str,
2522        lease_owner: &str,
2523        result: T,
2524        codec: &str,
2525    ) -> Result<Value> {
2526        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2527        let body = json!({
2528            "activity_attempt_id": activity_attempt_id,
2529            "lease_owner": lease_owner,
2530            "result": result
2531        });
2532        let path = format!("/worker/activity-tasks/{task_id}/complete");
2533        activity_task_response(
2534            self.request_json(
2535                reqwest::Method::POST,
2536                &path,
2537                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2538                Some(&body),
2539            )
2540            .await,
2541            "complete",
2542            task_id,
2543            activity_attempt_id,
2544        )
2545    }
2546
2547    pub async fn fail_activity_task(
2548        &self,
2549        task_id: &str,
2550        activity_attempt_id: &str,
2551        lease_owner: &str,
2552        message: impl Into<String>,
2553        non_retryable: bool,
2554    ) -> Result<Value> {
2555        let body = json!({
2556            "activity_attempt_id": activity_attempt_id,
2557            "lease_owner": lease_owner,
2558            "failure": {
2559                "message": message.into(),
2560                "type": "RustActivityFailure",
2561                "non_retryable": non_retryable
2562            }
2563        });
2564        let path = format!("/worker/activity-tasks/{task_id}/fail");
2565        activity_task_response(
2566            self.request_json(
2567                reqwest::Method::POST,
2568                &path,
2569                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2570                Some(&body),
2571            )
2572            .await,
2573            "fail",
2574            task_id,
2575            activity_attempt_id,
2576        )
2577    }
2578
2579    pub async fn heartbeat_activity_task<T: Serialize>(
2580        &self,
2581        task_id: &str,
2582        activity_attempt_id: &str,
2583        lease_owner: &str,
2584        details: T,
2585    ) -> Result<ActivityHeartbeatResponse> {
2586        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2587        let body = json!({
2588            "activity_attempt_id": activity_attempt_id,
2589            "lease_owner": lease_owner,
2590            "details": details
2591        });
2592        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2593        activity_task_response(
2594            self.request_json(
2595                reqwest::Method::POST,
2596                &path,
2597                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2598                Some(&body),
2599            )
2600            .await,
2601            "heartbeat",
2602            task_id,
2603            activity_attempt_id,
2604        )
2605    }
2606
2607    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2608        &self,
2609        method: reqwest::Method,
2610        path: &str,
2611        protocol: RequestProtocol,
2612        body: Option<&B>,
2613    ) -> Result<T> {
2614        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2615            .await
2616    }
2617
2618    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2619        &self,
2620        method: reqwest::Method,
2621        path: &str,
2622        protocol: RequestProtocol,
2623        body: Option<&B>,
2624        timeout: Duration,
2625    ) -> Result<T> {
2626        let mut request = self
2627            .http
2628            .request(method, format!("{}/api{}", self.base_url, path))
2629            .timeout(timeout)
2630            .header(reqwest::header::ACCEPT, "application/json")
2631            .header(reqwest::header::CONTENT_TYPE, "application/json")
2632            .header("X-Namespace", &self.namespace);
2633
2634        match protocol {
2635            RequestProtocol::Worker(version) => {
2636                request = request.header("X-Durable-Workflow-Protocol-Version", version);
2637            }
2638            RequestProtocol::ControlPlane => {
2639                request = request.header(
2640                    "X-Durable-Workflow-Control-Plane-Version",
2641                    CONTROL_PLANE_VERSION,
2642                );
2643            }
2644        }
2645
2646        if let Some(token) = self.auth_token(protocol.is_worker()) {
2647            request = request.bearer_auth(token);
2648        }
2649
2650        if let Some(body) = body {
2651            request = request.json(body);
2652        }
2653
2654        let response = request.send().await?;
2655        let status = response.status();
2656        let bytes = response.bytes().await?;
2657
2658        if !status.is_success() {
2659            let body = String::from_utf8_lossy(&bytes).to_string();
2660            if let Some(protocol) = protocol_failure(status, &body) {
2661                return Err(Error::Protocol(protocol));
2662            }
2663            return Err(Error::Http { status, body });
2664        }
2665
2666        if bytes.is_empty() {
2667            return Ok(serde_json::from_value(Value::Null)?);
2668        }
2669
2670        Ok(serde_json::from_slice(&bytes)?)
2671    }
2672
2673    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2674        &self,
2675        path: &str,
2676        protocol: RequestProtocol,
2677        body: &B,
2678        timeout: Duration,
2679        max_retries: usize,
2680    ) -> Result<T> {
2681        let mut retries = 0;
2682
2683        loop {
2684            let response = self
2685                .request_json_with_timeout(
2686                    reqwest::Method::POST,
2687                    path,
2688                    protocol,
2689                    Some(body),
2690                    timeout,
2691                )
2692                .await;
2693
2694            match response {
2695                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2696                response => return worker_poll_response(response),
2697            }
2698        }
2699    }
2700
2701    fn auth_token(&self, worker: bool) -> Option<&str> {
2702        if worker {
2703            self.worker_token
2704                .as_deref()
2705                .or(self.token.as_deref())
2706                .or(self.control_token.as_deref())
2707        } else {
2708            self.control_token
2709                .as_deref()
2710                .or(self.token.as_deref())
2711                .or(self.worker_token.as_deref())
2712        }
2713    }
2714}
2715
2716fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2717    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2718    let reason = body
2719        .get("reason")
2720        .and_then(Value::as_str)
2721        .unwrap_or("query_rejected")
2722        .to_string();
2723    let message = body
2724        .get("message")
2725        .or_else(|| body.get("error"))
2726        .and_then(Value::as_str)
2727        .unwrap_or("workflow query was rejected")
2728        .to_string();
2729
2730    QueryFailure {
2731        status: status.as_u16(),
2732        reason,
2733        message,
2734        body,
2735    }
2736}
2737
2738fn workflow_command_result(
2739    command: WorkflowCommandKind,
2740    data: Value,
2741    workflow_id: &str,
2742    run_id: Option<&str>,
2743) -> WorkflowCommandResult {
2744    WorkflowCommandResult {
2745        command,
2746        workflow_id: data
2747            .get("workflow_id")
2748            .and_then(Value::as_str)
2749            .unwrap_or(workflow_id)
2750            .to_string(),
2751        run_id: data
2752            .get("run_id")
2753            .and_then(Value::as_str)
2754            .or(run_id)
2755            .map(str::to_string),
2756        outcome: data
2757            .get("outcome")
2758            .and_then(Value::as_str)
2759            .map(str::to_string),
2760        reason: data
2761            .get("reason")
2762            .and_then(Value::as_str)
2763            .map(str::to_string),
2764        command_status: data
2765            .get("command_status")
2766            .and_then(Value::as_str)
2767            .map(str::to_string),
2768        raw: data,
2769    }
2770}
2771
2772fn workflow_command_rejection(
2773    command: WorkflowCommandKind,
2774    status: reqwest::StatusCode,
2775    raw_body: String,
2776    workflow_id: &str,
2777    run_id: Option<&str>,
2778) -> WorkflowCommandRejection {
2779    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2780    WorkflowCommandRejection {
2781        command,
2782        status: status.as_u16(),
2783        reason: body
2784            .get("reason")
2785            .and_then(Value::as_str)
2786            .unwrap_or("workflow_command_rejected")
2787            .to_string(),
2788        message: body
2789            .get("message")
2790            .or_else(|| body.get("error"))
2791            .and_then(Value::as_str)
2792            .unwrap_or("workflow lifecycle command was rejected")
2793            .to_string(),
2794        workflow_id: body
2795            .get("workflow_id")
2796            .and_then(Value::as_str)
2797            .unwrap_or(workflow_id)
2798            .to_string(),
2799        run_id: body
2800            .get("run_id")
2801            .and_then(Value::as_str)
2802            .or(run_id)
2803            .map(str::to_string),
2804        target_scope: body
2805            .get("target_scope")
2806            .and_then(Value::as_str)
2807            .map(str::to_string),
2808        body,
2809    }
2810}
2811
2812fn query_task_response(response: Result<Value>) -> Result<Value> {
2813    match response {
2814        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2815        response => response,
2816    }
2817}
2818
2819fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2820    match response {
2821        Err(Error::Http { status, body })
2822            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2823        {
2824            Ok(serde_json::from_str(&body)?)
2825        }
2826        response => response,
2827    }
2828}
2829
2830fn worker_poll_body_is_stop(body: &str) -> bool {
2831    serde_json::from_str::<Value>(body)
2832        .ok()
2833        .is_some_and(|body| {
2834            worker_poll_is_stop(
2835                body.get("poll_status").and_then(Value::as_str),
2836                body.get("reason").and_then(Value::as_str),
2837            )
2838        })
2839}
2840
2841fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2842    matches!(poll_status, Some("draining" | "stopped"))
2843        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2844}
2845
2846fn query_task_rejection_is_final(error: &Error) -> bool {
2847    matches!(
2848        error,
2849        Error::QueryFailed(failure)
2850            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2851    )
2852}
2853
2854fn activity_task_response<T>(
2855    response: Result<T>,
2856    operation: &str,
2857    task_id: &str,
2858    activity_attempt_id: &str,
2859) -> Result<T> {
2860    match response {
2861        Err(Error::Http { status, body }) => {
2862            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2863            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2864                operation: operation.to_string(),
2865                status: status.as_u16(),
2866                reason: body
2867                    .get("reason")
2868                    .and_then(Value::as_str)
2869                    .unwrap_or("activity_task_rejected")
2870                    .to_string(),
2871                task_id: body
2872                    .get("task_id")
2873                    .and_then(Value::as_str)
2874                    .unwrap_or(task_id)
2875                    .to_string(),
2876                activity_attempt_id: body
2877                    .get("activity_attempt_id")
2878                    .and_then(Value::as_str)
2879                    .unwrap_or(activity_attempt_id)
2880                    .to_string(),
2881                cancel_requested: body
2882                    .get("cancel_requested")
2883                    .and_then(Value::as_bool)
2884                    .unwrap_or(false),
2885                can_continue: body.get("can_continue").and_then(Value::as_bool),
2886                run_closed_reason: body
2887                    .get("run_closed_reason")
2888                    .and_then(Value::as_str)
2889                    .map(str::to_string),
2890                body,
2891            }))
2892        }
2893        response => response,
2894    }
2895}
2896
2897fn activity_task_rejection_is_final(error: &Error) -> bool {
2898    matches!(
2899        error,
2900        Error::ActivityTaskRejected(rejection)
2901            if matches!(
2902                rejection.reason.as_str(),
2903                "run_cancelled"
2904                    | "run_terminated"
2905                    | "attempt_closed"
2906                    | "stale_attempt"
2907                    | "activity_cancelled"
2908                    | "task_cancelled"
2909                    | "run_closed"
2910                    | "activity_not_running"
2911                    | "attempt_not_found"
2912            )
2913    )
2914}
2915
2916fn workflow_task_completion_is_terminal_timeout(
2917    error: &Error,
2918    task_id: &str,
2919    workflow_task_attempt: u64,
2920    run_id: Option<&str>,
2921) -> bool {
2922    let Error::Http { status, body } = error else {
2923        return false;
2924    };
2925    if *status != reqwest::StatusCode::CONFLICT {
2926        return false;
2927    }
2928
2929    let Some(run_id) = run_id else {
2930        return false;
2931    };
2932    let Ok(body) = serde_json::from_str::<Value>(body) else {
2933        return false;
2934    };
2935
2936    body.get("recorded").and_then(Value::as_bool) == Some(false)
2937        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2938        && body.get("run_status").and_then(Value::as_str) == Some("failed")
2939        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2940        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2941        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2942}
2943
2944fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2945    let body: Value = serde_json::from_str(raw_body).ok()?;
2946    let reason = body.get("reason")?.as_str()?;
2947    if !matches!(
2948        reason,
2949        "missing_protocol_version"
2950            | "unsupported_protocol_version"
2951            | "missing_control_plane_version"
2952            | "unsupported_control_plane_version"
2953    ) {
2954        return None;
2955    }
2956
2957    Some(ProtocolFailure {
2958        status: status.as_u16(),
2959        reason: reason.to_string(),
2960        message: body
2961            .get("message")
2962            .or_else(|| body.get("error"))
2963            .and_then(Value::as_str)
2964            .unwrap_or("protocol version rejected")
2965            .to_string(),
2966        supported_version: body
2967            .get("supported_version")
2968            .and_then(Value::as_str)
2969            .map(str::to_string),
2970        requested_version: body
2971            .get("requested_version")
2972            .and_then(Value::as_str)
2973            .map(str::to_string),
2974        body,
2975    })
2976}
2977
2978fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2979    timeout
2980        .as_secs()
2981        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2982        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2983}
2984
2985fn worker_operation_is_retryable(error: &Error) -> bool {
2986    match error {
2987        Error::Transport(error) => {
2988            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2989        }
2990        Error::Http { status, .. } => {
2991            matches!(
2992                *status,
2993                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2994            ) || status.is_server_error()
2995        }
2996        _ => false,
2997    }
2998}
2999
3000fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3001    let exponent = retry.saturating_sub(1).min(31) as u32;
3002    policy
3003        .initial_backoff
3004        .saturating_mul(1_u32 << exponent)
3005        .min(policy.max_backoff)
3006}
3007
3008#[derive(Debug)]
3009pub struct ClientBuilder {
3010    base_url: String,
3011    token: Option<String>,
3012    control_token: Option<String>,
3013    worker_token: Option<String>,
3014    namespace: String,
3015    timeout: Duration,
3016}
3017
3018impl ClientBuilder {
3019    pub fn token(mut self, token: Option<String>) -> Self {
3020        self.token = token;
3021        self
3022    }
3023
3024    pub fn control_token(mut self, token: Option<String>) -> Self {
3025        self.control_token = token;
3026        self
3027    }
3028
3029    pub fn worker_token(mut self, token: Option<String>) -> Self {
3030        self.worker_token = token;
3031        self
3032    }
3033
3034    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3035        self.namespace = namespace.into();
3036        self
3037    }
3038
3039    pub fn timeout(mut self, timeout: Duration) -> Self {
3040        self.timeout = timeout;
3041        self
3042    }
3043
3044    pub fn build(self) -> Result<Client> {
3045        let base_url = self.base_url.trim_end_matches('/').to_string();
3046        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3047            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3048            .unwrap_or_else(|_| base_url.ends_with("/api"));
3049
3050        if has_sdk_api_suffix {
3051            return Err(Error::InvalidBaseUrl);
3052        }
3053
3054        Ok(Client {
3055            http: reqwest::Client::builder().timeout(self.timeout).build()?,
3056            base_url,
3057            token: self.token,
3058            control_token: self.control_token,
3059            worker_token: self.worker_token,
3060            namespace: self.namespace,
3061        })
3062    }
3063}
3064
3065#[derive(Clone, Debug)]
3066pub struct WorkflowHandle {
3067    client: Client,
3068    pub workflow_id: String,
3069    pub run_id: Option<String>,
3070    pub workflow_type: String,
3071}
3072
3073impl WorkflowHandle {
3074    /// Describe whichever run is current for this stable workflow instance.
3075    pub async fn describe(&self) -> Result<WorkflowDescription> {
3076        self.client.describe_workflow(&self.workflow_id).await
3077    }
3078
3079    /// Describe the run identity originally selected by this handle.
3080    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3081        let run_id = self.run_id.as_deref().ok_or_else(|| {
3082            Error::Codec("run_id is required for selected-run description".to_string())
3083        })?;
3084        self.client
3085            .describe_workflow_run(&self.workflow_id, run_id)
3086            .await
3087    }
3088
3089    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3090        self.client
3091            .signal_workflow(&self.workflow_id, signal_name, input)
3092            .await
3093    }
3094
3095    /// Signal only if this handle's selected run is still current.
3096    pub async fn signal_selected_run<T: Serialize>(
3097        &self,
3098        signal_name: &str,
3099        input: T,
3100    ) -> Result<Value> {
3101        let run_id = self.run_id.as_deref().ok_or_else(|| {
3102            Error::Codec("run_id is required for selected-run signaling".to_string())
3103        })?;
3104        self.client
3105            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3106            .await
3107    }
3108
3109    /// Request cooperative cancellation of whichever run is current.
3110    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3111        self.client
3112            .cancel_workflow(&self.workflow_id, options)
3113            .await
3114    }
3115
3116    /// Request cancellation only if this handle's selected run is still current.
3117    pub async fn cancel_selected_run(
3118        &self,
3119        options: WorkflowCommandOptions,
3120    ) -> Result<WorkflowCommandResult> {
3121        let run_id = self.run_id.as_deref().ok_or_else(|| {
3122            Error::Codec("run_id is required for selected-run cancellation".to_string())
3123        })?;
3124        self.client
3125            .cancel_workflow_run(&self.workflow_id, run_id, options)
3126            .await
3127    }
3128
3129    /// Forcefully terminate whichever run is current.
3130    pub async fn terminate(
3131        &self,
3132        options: WorkflowCommandOptions,
3133    ) -> Result<WorkflowCommandResult> {
3134        self.client
3135            .terminate_workflow(&self.workflow_id, options)
3136            .await
3137    }
3138
3139    /// Terminate only if this handle's selected run is still current.
3140    pub async fn terminate_selected_run(
3141        &self,
3142        options: WorkflowCommandOptions,
3143    ) -> Result<WorkflowCommandResult> {
3144        let run_id = self.run_id.as_deref().ok_or_else(|| {
3145            Error::Codec("run_id is required for selected-run termination".to_string())
3146        })?;
3147        self.client
3148            .terminate_workflow_run(&self.workflow_id, run_id, options)
3149            .await
3150    }
3151
3152    /// Execute a named, read-only query against this workflow.
3153    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3154        self.client
3155            .query_workflow(&self.workflow_id, query_name, input)
3156            .await
3157    }
3158
3159    pub async fn query_avro_value<T: Serialize>(
3160        &self,
3161        query_name: &str,
3162        input: T,
3163    ) -> Result<AvroValue> {
3164        self.client
3165            .query_workflow_avro_value(&self.workflow_id, query_name, input)
3166            .await
3167    }
3168
3169    pub async fn update<T: Serialize>(
3170        &self,
3171        update_name: &str,
3172        input: T,
3173        request_id: Option<&str>,
3174    ) -> Result<Value> {
3175        self.client
3176            .update_workflow(&self.workflow_id, update_name, input, request_id)
3177            .await
3178    }
3179
3180    pub async fn update_avro_value<T: Serialize>(
3181        &self,
3182        update_name: &str,
3183        input: T,
3184        request_id: Option<&str>,
3185    ) -> Result<AvroValue> {
3186        self.client
3187            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3188            .await
3189    }
3190
3191    /// Query only if this handle's selected run is still current.
3192    pub async fn query_selected_run<T: Serialize>(
3193        &self,
3194        query_name: &str,
3195        input: T,
3196    ) -> Result<Value> {
3197        let run_id = self
3198            .run_id
3199            .as_deref()
3200            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3201        self.client
3202            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3203            .await
3204    }
3205
3206    /// Await the final terminal outcome of the current continue-as-new chain.
3207    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3208        self.result_target(options, None).await
3209    }
3210
3211    /// Await the final result without projecting Avro bytes through JSON.
3212    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3213        self.result_avro_value_target(options, None).await
3214    }
3215
3216    /// Await only the run identity originally selected by this handle.
3217    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3218        let run_id = self.run_id.as_deref().ok_or_else(|| {
3219            Error::Codec("run_id is required for selected-run result".to_string())
3220        })?;
3221        self.result_target(options, Some(run_id)).await
3222    }
3223
3224    /// Await the selected run's result on the lossless Avro Value surface.
3225    pub async fn result_selected_run_avro_value(
3226        &self,
3227        options: WorkflowResultOptions,
3228    ) -> Result<AvroValue> {
3229        let run_id = self.run_id.as_deref().ok_or_else(|| {
3230            Error::Codec("run_id is required for selected-run result".to_string())
3231        })?;
3232        self.result_avro_value_target(options, Some(run_id)).await
3233    }
3234
3235    async fn result_avro_value_target(
3236        &self,
3237        options: WorkflowResultOptions,
3238        selected_run_id: Option<&str>,
3239    ) -> Result<AvroValue> {
3240        let started = Instant::now();
3241
3242        loop {
3243            let description = match selected_run_id {
3244                Some(run_id) => {
3245                    self.client
3246                        .describe_workflow_run(&self.workflow_id, run_id)
3247                        .await?
3248                }
3249                None => self.describe().await?,
3250            };
3251            if description.is_completed() {
3252                return description.output_avro_value.ok_or_else(|| {
3253                    Error::Codec(
3254                        "missing_payload_envelope: typed workflow result requires output_envelope"
3255                            .to_string(),
3256                    )
3257                });
3258            }
3259            if description.is_terminal() {
3260                let outcome =
3261                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3262                return Err(match outcome.kind {
3263                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3264                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3265                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3266                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3267                });
3268            }
3269            if started.elapsed() >= options.timeout {
3270                return Err(Error::Timeout);
3271            }
3272            tokio::time::sleep(options.poll_interval).await;
3273        }
3274    }
3275
3276    async fn result_target(
3277        &self,
3278        options: WorkflowResultOptions,
3279        selected_run_id: Option<&str>,
3280    ) -> Result<Value> {
3281        let started = Instant::now();
3282
3283        loop {
3284            let description = match selected_run_id {
3285                Some(run_id) => {
3286                    self.client
3287                        .describe_workflow_run(&self.workflow_id, run_id)
3288                        .await?
3289                }
3290                None => self.describe().await?,
3291            };
3292            if description.is_completed() {
3293                return Ok(description.output.unwrap_or(Value::Null));
3294            }
3295
3296            if description.is_terminal() {
3297                let outcome =
3298                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3299                return Err(match outcome.kind {
3300                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3301                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3302                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3303                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3304                });
3305            }
3306
3307            if started.elapsed() >= options.timeout {
3308                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3309                    kind: WorkflowTerminalKind::TimedOut,
3310                    workflow_id: description
3311                        .workflow_id
3312                        .clone()
3313                        .unwrap_or_else(|| self.workflow_id.clone()),
3314                    run_id: description
3315                        .run_id
3316                        .clone()
3317                        .or_else(|| selected_run_id.map(str::to_string)),
3318                    reason: "result_wait_timeout".to_string(),
3319                    failure_category: Some("client_timeout".to_string()),
3320                    failure_id: None,
3321                    exception_type: None,
3322                    exception_class: None,
3323                    non_retryable: None,
3324                    message: Some(format!(
3325                        "workflow result was not terminal within {:?}",
3326                        options.timeout
3327                    )),
3328                    exception: None,
3329                    raw: description.raw_value(),
3330                }));
3331            }
3332
3333            tokio::time::sleep(options.poll_interval).await;
3334        }
3335    }
3336}
3337
3338#[derive(Clone, Copy, Debug)]
3339pub struct WorkflowResultOptions {
3340    pub poll_interval: Duration,
3341    pub timeout: Duration,
3342}
3343
3344impl Default for WorkflowResultOptions {
3345    fn default() -> Self {
3346        Self {
3347            poll_interval: Duration::from_millis(500),
3348            timeout: Duration::from_secs(30),
3349        }
3350    }
3351}
3352
3353#[derive(Clone, Debug, Deserialize)]
3354pub struct WorkflowDescription {
3355    pub workflow_id: Option<String>,
3356    pub run_id: Option<String>,
3357    pub workflow_type: Option<String>,
3358    pub status: Option<String>,
3359    #[serde(default)]
3360    pub closed_reason: Option<String>,
3361    #[serde(default)]
3362    pub error: Option<String>,
3363    #[serde(default)]
3364    pub failure: Option<Value>,
3365    #[serde(default)]
3366    pub exception: Option<Value>,
3367    #[serde(default)]
3368    pub failures: Vec<Value>,
3369    #[serde(default)]
3370    pub output: Option<Value>,
3371    #[serde(default)]
3372    pub output_envelope: Option<Value>,
3373    #[serde(skip)]
3374    pub output_avro_value: Option<AvroValue>,
3375    #[serde(flatten)]
3376    pub raw: HashMap<String, Value>,
3377}
3378
3379impl WorkflowDescription {
3380    pub fn is_completed(&self) -> bool {
3381        matches!(self.status.as_deref(), Some("completed" | "Completed"))
3382    }
3383
3384    pub fn is_terminal(&self) -> bool {
3385        matches!(
3386            self.status.as_deref(),
3387            Some(
3388                "completed"
3389                    | "Completed"
3390                    | "failed"
3391                    | "Failed"
3392                    | "cancelled"
3393                    | "Cancelled"
3394                    | "terminated"
3395                    | "Terminated"
3396                    | "timed_out"
3397                    | "TimedOut",
3398            )
3399        )
3400    }
3401
3402    fn decode_payloads(&mut self) -> Result<()> {
3403        if let Some(envelope) = &self.output_envelope {
3404            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3405            self.output = Some(value.clone().into_json()?);
3406            self.output_avro_value = Some(value);
3407        }
3408
3409        Ok(())
3410    }
3411
3412    fn raw_value(&self) -> Value {
3413        let mut data = self.raw.clone();
3414        data.insert(
3415            "workflow_id".to_string(),
3416            self.workflow_id
3417                .clone()
3418                .map(Value::String)
3419                .unwrap_or(Value::Null),
3420        );
3421        data.insert(
3422            "run_id".to_string(),
3423            self.run_id
3424                .clone()
3425                .map(Value::String)
3426                .unwrap_or(Value::Null),
3427        );
3428        data.insert(
3429            "workflow_type".to_string(),
3430            self.workflow_type
3431                .clone()
3432                .map(Value::String)
3433                .unwrap_or(Value::Null),
3434        );
3435        data.insert(
3436            "status".to_string(),
3437            self.status
3438                .clone()
3439                .map(Value::String)
3440                .unwrap_or(Value::Null),
3441        );
3442        data.insert(
3443            "closed_reason".to_string(),
3444            self.closed_reason
3445                .clone()
3446                .map(Value::String)
3447                .unwrap_or(Value::Null),
3448        );
3449        if let Some(failure) = &self.failure {
3450            data.insert("failure".to_string(), failure.clone());
3451        }
3452        if let Some(exception) = &self.exception {
3453            data.insert("exception".to_string(), exception.clone());
3454        }
3455        Value::Object(data.into_iter().collect())
3456    }
3457}
3458
3459fn workflow_terminal_outcome(
3460    description: &WorkflowDescription,
3461    workflow_id: &str,
3462    run_id: Option<&str>,
3463) -> WorkflowTerminalOutcome {
3464    let terminal_kind = description
3465        .closed_reason
3466        .as_deref()
3467        .or(description.status.as_deref())
3468        .unwrap_or("failed")
3469        .to_ascii_lowercase();
3470    let kind = match terminal_kind.as_str() {
3471        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3472        "terminated" => WorkflowTerminalKind::Terminated,
3473        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3474        _ => WorkflowTerminalKind::Failed,
3475    };
3476    let default_reason = match kind {
3477        WorkflowTerminalKind::Failed => "workflow_failed",
3478        WorkflowTerminalKind::Cancelled => "cancelled",
3479        WorkflowTerminalKind::Terminated => "terminated",
3480        WorkflowTerminalKind::TimedOut => "timed_out",
3481    };
3482    let failure = description
3483        .failure
3484        .as_ref()
3485        .filter(|value| value.is_object());
3486    let nested_failure = failure
3487        .and_then(|value| value.get("failures"))
3488        .and_then(Value::as_array)
3489        .and_then(|failures| failures.last())
3490        .or_else(|| description.failures.last());
3491    let exception = description
3492        .exception
3493        .clone()
3494        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3495        .or_else(|| {
3496            nested_failure
3497                .and_then(|value| value.get("exception_payload"))
3498                .cloned()
3499        });
3500    let string_field = |name: &str| {
3501        failure
3502            .and_then(|value| value.get(name))
3503            .and_then(Value::as_str)
3504            .or_else(|| {
3505                nested_failure
3506                    .and_then(|value| value.get(name))
3507                    .and_then(Value::as_str)
3508            })
3509            .map(str::to_string)
3510    };
3511    let exception_field = |name: &str| {
3512        exception
3513            .as_ref()
3514            .and_then(|value| value.get(name))
3515            .and_then(Value::as_str)
3516            .map(str::to_string)
3517    };
3518    let message = description
3519        .error
3520        .clone()
3521        .or_else(|| string_field("message"))
3522        .or_else(|| exception_field("message"));
3523    let reason = description
3524        .raw
3525        .get("reason")
3526        .and_then(Value::as_str)
3527        .map(str::to_string)
3528        .or_else(|| {
3529            failure
3530                .and_then(|value| value.get("reason"))
3531                .and_then(Value::as_str)
3532                .map(str::to_string)
3533        })
3534        .or_else(|| description.closed_reason.clone())
3535        .unwrap_or_else(|| default_reason.to_string());
3536    let failure_id = string_field("failure_id").or_else(|| {
3537        nested_failure
3538            .and_then(|value| value.get("id"))
3539            .and_then(Value::as_str)
3540            .map(str::to_string)
3541    });
3542
3543    WorkflowTerminalOutcome {
3544        kind,
3545        workflow_id: description
3546            .workflow_id
3547            .clone()
3548            .unwrap_or_else(|| workflow_id.to_string()),
3549        run_id: description
3550            .run_id
3551            .clone()
3552            .or_else(|| run_id.map(str::to_string)),
3553        reason,
3554        failure_category: string_field("failure_category")
3555            .or_else(|| Some(default_reason.to_string())),
3556        failure_id,
3557        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3558        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3559        non_retryable: failure
3560            .and_then(|value| value.get("non_retryable"))
3561            .and_then(Value::as_bool)
3562            .or_else(|| {
3563                nested_failure
3564                    .and_then(|value| value.get("non_retryable"))
3565                    .and_then(Value::as_bool)
3566            }),
3567        message,
3568        exception,
3569        raw: description.raw_value(),
3570    }
3571}
3572
3573#[derive(Clone, Debug, Deserialize)]
3574pub struct RegisterWorkerResponse {
3575    pub worker_id: String,
3576    pub registered: bool,
3577    #[serde(default)]
3578    pub heartbeat_interval_seconds: Option<u64>,
3579    #[serde(default)]
3580    pub protocol_version: Option<String>,
3581    #[serde(default)]
3582    pub server_capabilities: Option<Value>,
3583}
3584
3585#[derive(Clone, Debug, Deserialize)]
3586pub struct PollWorkflowTaskResponse {
3587    #[serde(default)]
3588    pub task: Option<WorkflowTask>,
3589    #[serde(default)]
3590    pub poll_status: Option<String>,
3591    #[serde(default)]
3592    pub reason: Option<String>,
3593    #[serde(default)]
3594    pub protocol_version: Option<String>,
3595    #[serde(default)]
3596    pub server_capabilities: Option<Value>,
3597}
3598
3599impl PollWorkflowTaskResponse {
3600    /// Classify this response without parsing server display text.
3601    pub fn outcome(&self) -> WorkerPollOutcome {
3602        worker_poll_outcome(
3603            self.task.is_some(),
3604            self.poll_status.as_deref(),
3605            self.reason.as_deref(),
3606        )
3607    }
3608}
3609
3610#[derive(Clone, Debug, Deserialize)]
3611pub struct PollActivityTaskResponse {
3612    #[serde(default)]
3613    pub task: Option<ActivityTask>,
3614    #[serde(default)]
3615    pub poll_status: Option<String>,
3616    #[serde(default)]
3617    pub reason: Option<String>,
3618}
3619
3620impl PollActivityTaskResponse {
3621    /// Classify this response without parsing server display text.
3622    pub fn outcome(&self) -> WorkerPollOutcome {
3623        worker_poll_outcome(
3624            self.task.is_some(),
3625            self.poll_status.as_deref(),
3626            self.reason.as_deref(),
3627        )
3628    }
3629}
3630
3631#[derive(Clone, Debug, Deserialize)]
3632pub struct PollQueryTaskResponse {
3633    #[serde(default)]
3634    pub task: Option<QueryTask>,
3635    #[serde(default)]
3636    pub poll_status: Option<String>,
3637    #[serde(default)]
3638    pub reason: Option<String>,
3639}
3640
3641impl PollQueryTaskResponse {
3642    /// Classify this response without parsing server display text.
3643    pub fn outcome(&self) -> WorkerPollOutcome {
3644        worker_poll_outcome(
3645            self.task.is_some(),
3646            self.poll_status.as_deref(),
3647            self.reason.as_deref(),
3648        )
3649    }
3650}
3651
3652/// Stable classification for worker poll responses.
3653#[derive(Clone, Debug, PartialEq, Eq)]
3654pub enum WorkerPollOutcome {
3655    /// A task was leased and is available on the response.
3656    Task,
3657    /// No task was leased, but the worker should continue polling.
3658    Idle {
3659        poll_status: Option<String>,
3660        reason: Option<String>,
3661    },
3662    /// The server asked this worker to stop claiming new work.
3663    Stop {
3664        poll_status: Option<String>,
3665        reason: Option<String>,
3666    },
3667}
3668
3669impl WorkerPollOutcome {
3670    pub fn should_stop(&self) -> bool {
3671        matches!(self, Self::Stop { .. })
3672    }
3673}
3674
3675fn worker_poll_outcome(
3676    has_task: bool,
3677    poll_status: Option<&str>,
3678    reason: Option<&str>,
3679) -> WorkerPollOutcome {
3680    if worker_poll_is_stop(poll_status, reason) {
3681        return WorkerPollOutcome::Stop {
3682            poll_status: poll_status.map(str::to_string),
3683            reason: reason.map(str::to_string),
3684        };
3685    }
3686
3687    if has_task {
3688        WorkerPollOutcome::Task
3689    } else {
3690        WorkerPollOutcome::Idle {
3691            poll_status: poll_status.map(str::to_string),
3692            reason: reason.map(str::to_string),
3693        }
3694    }
3695}
3696
3697/// An ephemeral server-routed query task.
3698#[derive(Clone, Debug, Deserialize)]
3699pub struct QueryTask {
3700    pub query_task_id: String,
3701    #[serde(default = "default_workflow_task_attempt")]
3702    pub query_task_attempt: u64,
3703    #[serde(default)]
3704    pub lease_owner: Option<String>,
3705    #[serde(default)]
3706    pub workflow_id: Option<String>,
3707    #[serde(default)]
3708    pub run_id: Option<String>,
3709    pub workflow_type: String,
3710    pub query_name: String,
3711    #[serde(default = "default_payload_codec")]
3712    pub payload_codec: String,
3713    #[serde(default)]
3714    pub workflow_arguments: Option<Value>,
3715    #[serde(default)]
3716    pub query_arguments: Option<Value>,
3717    #[serde(default)]
3718    pub history_events: Vec<HistoryEvent>,
3719    #[serde(default)]
3720    pub history_export: Option<Value>,
3721    #[serde(default)]
3722    pub run_status: Option<String>,
3723}
3724
3725#[derive(Clone, Debug, Deserialize)]
3726pub struct WorkflowTask {
3727    pub task_id: String,
3728    #[serde(default)]
3729    pub workflow_id: Option<String>,
3730    #[serde(default)]
3731    pub run_id: Option<String>,
3732    pub workflow_type: String,
3733    #[serde(default = "default_payload_codec")]
3734    pub payload_codec: String,
3735    #[serde(default)]
3736    pub arguments: Option<Value>,
3737    #[serde(default)]
3738    pub history_events: Vec<HistoryEvent>,
3739    #[serde(default)]
3740    pub total_history_events: Option<u64>,
3741    #[serde(default)]
3742    pub history_size_bytes: Option<u64>,
3743    #[serde(default)]
3744    pub continue_as_new_recommended: Option<bool>,
3745    #[serde(default)]
3746    pub history_budget_pressure: Option<String>,
3747    #[serde(default)]
3748    pub next_history_page_token: Option<String>,
3749    #[serde(default = "default_workflow_task_attempt")]
3750    pub workflow_task_attempt: u64,
3751    #[serde(default)]
3752    pub workflow_signal_id: Option<String>,
3753    #[serde(default)]
3754    pub signal_name: Option<String>,
3755    #[serde(default)]
3756    pub signal_arguments: Option<Value>,
3757    #[serde(default)]
3758    pub workflow_update_id: Option<String>,
3759    #[serde(default)]
3760    pub update_name: Option<String>,
3761    #[serde(default)]
3762    pub lease_owner: Option<String>,
3763}
3764
3765impl WorkflowTask {
3766    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3767        self.history_events.extend(page.history_events);
3768
3769        if page.total_history_events.is_some() {
3770            self.total_history_events = page.total_history_events;
3771        }
3772
3773        self.next_history_page_token = page
3774            .next_history_page_token
3775            .filter(|token| !token.is_empty());
3776    }
3777}
3778
3779#[derive(Clone, Debug, Deserialize)]
3780struct WorkflowTaskHistoryPage {
3781    #[serde(default)]
3782    history_events: Vec<HistoryEvent>,
3783    #[serde(default)]
3784    total_history_events: Option<u64>,
3785    #[serde(default)]
3786    next_history_page_token: Option<String>,
3787}
3788
3789#[derive(Clone, Debug, Deserialize)]
3790pub struct ActivityTask {
3791    pub task_id: String,
3792    #[serde(default)]
3793    pub activity_attempt_id: Option<String>,
3794    #[serde(default)]
3795    pub attempt_id: Option<String>,
3796    pub activity_type: String,
3797    #[serde(default = "default_payload_codec")]
3798    pub payload_codec: String,
3799    #[serde(default)]
3800    pub arguments: Option<Value>,
3801    #[serde(default = "default_attempt_number")]
3802    pub attempt_number: u64,
3803    #[serde(default)]
3804    pub lease_owner: Option<String>,
3805}
3806
3807#[derive(Clone, Debug, Deserialize)]
3808pub struct HistoryEvent {
3809    #[serde(alias = "type")]
3810    pub event_type: String,
3811    #[serde(default)]
3812    pub payload: Value,
3813    #[serde(flatten)]
3814    pub raw: HashMap<String, Value>,
3815}
3816
3817/// One decoded signal in the committed workflow-history snapshot.
3818#[derive(Clone, Debug, PartialEq)]
3819pub struct QuerySignal {
3820    pub id: Option<String>,
3821    pub name: String,
3822    pub arguments: Vec<Value>,
3823    avro_arguments: Vec<AvroValue>,
3824    pub workflow_sequence: Option<u64>,
3825}
3826
3827impl QuerySignal {
3828    /// Lossless fixed Avro Value arguments for this committed signal.
3829    pub fn arguments_avro_value(&self) -> &[AvroValue] {
3830        &self.avro_arguments
3831    }
3832}
3833
3834/// Immutable state supplied to a registered query handler.
3835///
3836/// This context intentionally exposes no activity, signal-wait, or command
3837/// APIs. Query handlers inspect committed history and return a value; query
3838/// completion does not append an event or advance deterministic execution.
3839#[derive(Clone, Debug)]
3840pub struct QueryContext {
3841    pub workflow_id: Option<String>,
3842    pub run_id: Option<String>,
3843    pub workflow_type: String,
3844    pub run_status: Option<String>,
3845    workflow_input: Value,
3846    workflow_input_avro_value: AvroValue,
3847    history_events: Arc<Vec<HistoryEvent>>,
3848    signal_events: Arc<Vec<QuerySignal>>,
3849}
3850
3851impl QueryContext {
3852    /// The normalized argument list used to start the workflow.
3853    pub fn workflow_input(&self) -> &Value {
3854        &self.workflow_input
3855    }
3856
3857    /// The lossless fixed Avro Value argument list used to start the workflow.
3858    pub fn workflow_input_avro_value(&self) -> &AvroValue {
3859        &self.workflow_input_avro_value
3860    }
3861
3862    /// The immutable committed history used for this query snapshot.
3863    pub fn history_events(&self) -> &[HistoryEvent] {
3864        self.history_events.as_slice()
3865    }
3866
3867    /// All decoded signals in committed workflow order.
3868    pub fn signal_events(&self) -> &[QuerySignal] {
3869        self.signal_events.as_slice()
3870    }
3871
3872    /// Decoded argument lists for each committed signal with `signal_name`.
3873    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3874        self.signal_events
3875            .iter()
3876            .filter(|signal| signal.name == signal_name)
3877            .map(|signal| signal.arguments.clone())
3878            .collect()
3879    }
3880
3881    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
3882    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3883        self.signal_events
3884            .iter()
3885            .filter(|signal| signal.name == signal_name)
3886            .map(|signal| signal.avro_arguments.clone())
3887            .collect()
3888    }
3889}
3890
3891#[derive(Clone, Debug, Deserialize)]
3892pub struct ActivityHeartbeatResponse {
3893    #[serde(default)]
3894    pub cancel_requested: bool,
3895    #[serde(default)]
3896    pub heartbeat_recorded: bool,
3897    #[serde(default)]
3898    pub can_continue: Option<bool>,
3899    #[serde(default)]
3900    pub reason: Option<String>,
3901    #[serde(default)]
3902    pub run_closed_reason: Option<String>,
3903    #[serde(default)]
3904    pub run_closed_at: Option<String>,
3905    #[serde(default)]
3906    pub lease_expires_at: Option<String>,
3907    #[serde(default)]
3908    pub last_heartbeat_at: Option<String>,
3909}
3910
3911impl ActivityHeartbeatResponse {
3912    /// Whether the activity should stop instead of attempting completion.
3913    pub fn should_stop(&self) -> bool {
3914        self.cancel_requested || self.can_continue == Some(false)
3915    }
3916}
3917
3918fn default_payload_codec() -> String {
3919    DEFAULT_CODEC.to_string()
3920}
3921
3922fn default_workflow_task_attempt() -> u64 {
3923    1
3924}
3925
3926fn default_attempt_number() -> u64 {
3927    1
3928}
3929
3930type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3931type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3932type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3933type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3934type ReplayedWorkflowHandler =
3935    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3936type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3937type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3938type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3939type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3940type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3941type ReplayedQueryHandler = Arc<
3942    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3943        + Send
3944        + Sync,
3945>;
3946type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3947
3948struct ReplayedWorkflowInvocation {
3949    future: WorkflowFuture,
3950    snapshot: WorkflowStateSnapshot,
3951}
3952
3953#[derive(Clone)]
3954struct RegisteredWorkflow {
3955    execute: WorkflowHandler,
3956    replay: Option<ReplayedWorkflowHandler>,
3957    state_type: Option<TypeId>,
3958}
3959
3960#[derive(Clone)]
3961enum RegisteredQuery {
3962    Snapshot(QueryHandler),
3963    Replayed {
3964        state_type: TypeId,
3965        handler: ReplayedQueryHandler,
3966    },
3967}
3968
3969#[derive(Clone, Debug)]
3970pub struct WorkerHeartbeatObservation {
3971    pub worker_id: String,
3972    pub task_queue: String,
3973    pub acknowledged_at_unix_millis: u64,
3974    pub acknowledgement: Value,
3975}
3976
3977/// Bounded retry policy for worker poll acquisition and worker heartbeats.
3978///
3979/// Expected empty long polls are normal successful responses. Transport
3980/// failures, HTTP 408/429 responses, and server errors are retried with capped
3981/// exponential backoff. Authentication, protocol, codec, and handler failures
3982/// are never retried by the worker.
3983#[derive(Clone, Copy, Debug)]
3984pub struct WorkerRetryPolicy {
3985    /// Number of retries after the initial request fails.
3986    pub max_retries: usize,
3987    /// Delay before the first retry.
3988    pub initial_backoff: Duration,
3989    /// Maximum delay between retries.
3990    pub max_backoff: Duration,
3991}
3992
3993impl Default for WorkerRetryPolicy {
3994    fn default() -> Self {
3995        Self {
3996            max_retries: 5,
3997            initial_backoff: Duration::from_millis(100),
3998            max_backoff: Duration::from_secs(5),
3999        }
4000    }
4001}
4002
4003#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4004enum ManagedPollOutcome {
4005    Idle,
4006    Handled,
4007    Stop,
4008}
4009
4010#[derive(Clone)]
4011pub struct Worker {
4012    client: Client,
4013    worker_id: String,
4014    task_queue: String,
4015    workflows: HashMap<String, RegisteredWorkflow>,
4016    activities: HashMap<String, ActivityHandler>,
4017    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4018    updates: HashMap<String, HashMap<String, UpdateHandler>>,
4019    max_concurrent_workflow_tasks: usize,
4020    max_concurrent_activity_tasks: usize,
4021    poll_timeout: Duration,
4022    heartbeat_interval: Duration,
4023    retry_policy: WorkerRetryPolicy,
4024    heartbeat_observer: Option<WorkerHeartbeatObserver>,
4025}
4026
4027impl Worker {
4028    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4029        Self {
4030            client,
4031            worker_id: default_worker_id(),
4032            task_queue: task_queue.into(),
4033            workflows: HashMap::new(),
4034            activities: HashMap::new(),
4035            queries: HashMap::new(),
4036            updates: HashMap::new(),
4037            max_concurrent_workflow_tasks: 10,
4038            max_concurrent_activity_tasks: 10,
4039            poll_timeout: Duration::from_secs(30),
4040            heartbeat_interval: Duration::from_secs(60),
4041            retry_policy: WorkerRetryPolicy::default(),
4042            heartbeat_observer: None,
4043        }
4044    }
4045
4046    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4047        self.worker_id = worker_id.into();
4048        self
4049    }
4050
4051    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4052        self.poll_timeout = timeout;
4053        self
4054    }
4055
4056    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4057        self.heartbeat_interval = interval;
4058        self
4059    }
4060
4061    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
4062    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4063        self.retry_policy = policy;
4064        self
4065    }
4066
4067    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4068    where
4069        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4070    {
4071        self.heartbeat_observer = Some(Arc::new(observer));
4072        self
4073    }
4074
4075    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4076        self.max_concurrent_workflow_tasks = count.max(1);
4077        self
4078    }
4079
4080    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4081        self.max_concurrent_activity_tasks = count.max(1);
4082        self
4083    }
4084
4085    /// Register a workflow handler.
4086    ///
4087    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
4088    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
4089    /// while acquiring or decoding a worker task remain worker-operation
4090    /// failures and do not get converted into workflow outcomes.
4091    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4092    where
4093        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4094        Fut: Future<Output = Result<Value>> + Send + 'static,
4095    {
4096        let handler = Arc::new(handler);
4097        self.workflows.insert(
4098            workflow_type.into(),
4099            RegisteredWorkflow {
4100                execute: Arc::new(move |ctx, input| {
4101                    let handler = Arc::clone(&handler);
4102                    Box::pin(async move {
4103                        let result = handler(ctx, input.into_json()?).await?;
4104                        AvroValue::from_serialize(&result)
4105                    })
4106                }),
4107                replay: None,
4108                state_type: None,
4109            },
4110        );
4111    }
4112
4113    /// Register a workflow on the lossless fixed Avro Value surface.
4114    pub fn register_workflow_avro_value<F, Fut>(
4115        &mut self,
4116        workflow_type: impl Into<String>,
4117        handler: F,
4118    ) where
4119        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4120        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4121    {
4122        self.workflows.insert(
4123            workflow_type.into(),
4124            RegisteredWorkflow {
4125                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4126                replay: None,
4127                state_type: None,
4128            },
4129        );
4130    }
4131
4132    /// Register a workflow whose typed instance state can be reconstructed for queries.
4133    ///
4134    /// `state_factory` creates a fresh instance for every normal workflow task and
4135    /// query replay. The workflow handler is the single source of truth for state
4136    /// transitions: it updates [`WorkflowInstance`] after activities and signals
4137    /// resolve. Query replay runs this same handler over committed history and
4138    /// discards any commands it would emit.
4139    pub fn register_replayed_workflow<S, Factory, F, Fut>(
4140        &mut self,
4141        workflow_type: impl Into<String>,
4142        state_factory: Factory,
4143        handler: F,
4144    ) where
4145        S: Clone + Send + Sync + 'static,
4146        Factory: Fn() -> S + Send + Sync + 'static,
4147        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4148        Fut: Future<Output = Result<Value>> + Send + 'static,
4149    {
4150        let state_factory = Arc::new(state_factory);
4151        let handler = Arc::new(handler);
4152
4153        let execute_factory = Arc::clone(&state_factory);
4154        let execute_handler = Arc::clone(&handler);
4155        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4156            let state = WorkflowInstance::new(execute_factory());
4157            let handler = Arc::clone(&execute_handler);
4158            Box::pin(async move {
4159                let result = handler(ctx, input.into_json()?, state).await?;
4160                AvroValue::from_serialize(&result)
4161            }) as WorkflowFuture
4162        });
4163
4164        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4165            let state = WorkflowInstance::new(state_factory());
4166            let snapshot_state = state.clone();
4167            let snapshot: WorkflowStateSnapshot =
4168                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4169            let replay_handler = Arc::clone(&handler);
4170            let future = async move {
4171                let result = replay_handler(ctx, input.into_json()?, state).await?;
4172                AvroValue::from_serialize(&result)
4173            };
4174            ReplayedWorkflowInvocation {
4175                future: Box::pin(future),
4176                snapshot,
4177            }
4178        });
4179
4180        self.workflows.insert(
4181            workflow_type.into(),
4182            RegisteredWorkflow {
4183                execute,
4184                replay: Some(replay),
4185                state_type: Some(TypeId::of::<S>()),
4186            },
4187        );
4188    }
4189
4190    /// Register a replayable workflow on the lossless fixed Avro Value surface.
4191    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4192        &mut self,
4193        workflow_type: impl Into<String>,
4194        state_factory: Factory,
4195        handler: F,
4196    ) where
4197        S: Clone + Send + Sync + 'static,
4198        Factory: Fn() -> S + Send + Sync + 'static,
4199        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4200        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4201    {
4202        let state_factory = Arc::new(state_factory);
4203        let handler = Arc::new(handler);
4204
4205        let execute_factory = Arc::clone(&state_factory);
4206        let execute_handler = Arc::clone(&handler);
4207        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4208            let state = WorkflowInstance::new(execute_factory());
4209            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4210        });
4211
4212        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4213            let state = WorkflowInstance::new(state_factory());
4214            let snapshot_state = state.clone();
4215            let snapshot: WorkflowStateSnapshot =
4216                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4217            ReplayedWorkflowInvocation {
4218                future: Box::pin(handler(ctx, input, state)),
4219                snapshot,
4220            }
4221        });
4222
4223        self.workflows.insert(
4224            workflow_type.into(),
4225            RegisteredWorkflow {
4226                execute,
4227                replay: Some(replay),
4228                state_type: Some(TypeId::of::<S>()),
4229            },
4230        );
4231    }
4232
4233    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4234    where
4235        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4236        Fut: Future<Output = Result<Value>> + Send + 'static,
4237    {
4238        let handler = Arc::new(handler);
4239        self.activities.insert(
4240            activity_type.into(),
4241            Arc::new(move |ctx, args| {
4242                let handler = Arc::clone(&handler);
4243                Box::pin(async move {
4244                    let result = handler(ctx, args.into_json()?).await?;
4245                    AvroValue::from_serialize(&result)
4246                })
4247            }),
4248        );
4249    }
4250
4251    /// Register an activity on the lossless fixed Avro Value surface.
4252    pub fn register_activity_avro_value<F, Fut>(
4253        &mut self,
4254        activity_type: impl Into<String>,
4255        handler: F,
4256    ) where
4257        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4258        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4259    {
4260        self.activities.insert(
4261            activity_type.into(),
4262            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4263        );
4264    }
4265
4266    /// Register a named, read-only query handler for a workflow type.
4267    ///
4268    /// The workflow type must also be registered with [`Worker::register_workflow`]
4269    /// before the worker runs. The handler receives only an immutable committed
4270    /// state snapshot and normalized query arguments.
4271    pub fn register_query<F, Fut>(
4272        &mut self,
4273        workflow_type: impl Into<String>,
4274        query_name: impl Into<String>,
4275        handler: F,
4276    ) where
4277        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4278        Fut: Future<Output = Result<Value>> + Send + 'static,
4279    {
4280        let handler = Arc::new(handler);
4281        self.queries
4282            .entry(workflow_type.into())
4283            .or_default()
4284            .insert(
4285                query_name.into(),
4286                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4287                    let handler = Arc::clone(&handler);
4288                    Box::pin(async move {
4289                        let result = handler(ctx, args.into_json()?).await?;
4290                        AvroValue::from_serialize(&result)
4291                    })
4292                })),
4293            );
4294    }
4295
4296    /// Register a query handler on the lossless fixed Avro Value surface.
4297    pub fn register_query_avro_value<F, Fut>(
4298        &mut self,
4299        workflow_type: impl Into<String>,
4300        query_name: impl Into<String>,
4301        handler: F,
4302    ) where
4303        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4304        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4305    {
4306        self.queries
4307            .entry(workflow_type.into())
4308            .or_default()
4309            .insert(
4310                query_name.into(),
4311                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4312            );
4313    }
4314
4315    /// Register a named query against deterministically replayed instance state.
4316    ///
4317    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
4318    /// same state type `S`. The handler receives an immutable, detached state
4319    /// clone, so successful and failed queries cannot affect workflow execution
4320    /// or the state reconstructed by a later query.
4321    pub fn register_replayed_query<S, F, Fut>(
4322        &mut self,
4323        workflow_type: impl Into<String>,
4324        query_name: impl Into<String>,
4325        handler: F,
4326    ) where
4327        S: Clone + Send + Sync + 'static,
4328        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4329        Fut: Future<Output = Result<Value>> + Send + 'static,
4330    {
4331        let handler = Arc::new(handler);
4332        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4333            let state = state.downcast::<S>().map_err(|_| {
4334                "registered query state type does not match the replayed workflow state".to_string()
4335            })?;
4336            let handler = Arc::clone(&handler);
4337            Ok(Box::pin(async move {
4338                let result = handler(ctx, state, args.into_json()?).await?;
4339                AvroValue::from_serialize(&result)
4340            }))
4341        });
4342
4343        self.queries
4344            .entry(workflow_type.into())
4345            .or_default()
4346            .insert(
4347                query_name.into(),
4348                RegisteredQuery::Replayed {
4349                    state_type: TypeId::of::<S>(),
4350                    handler: erased_handler,
4351                },
4352            );
4353    }
4354
4355    /// Register a replayed-state query on the lossless fixed Avro Value surface.
4356    pub fn register_replayed_query_avro_value<S, F, Fut>(
4357        &mut self,
4358        workflow_type: impl Into<String>,
4359        query_name: impl Into<String>,
4360        handler: F,
4361    ) where
4362        S: Clone + Send + Sync + 'static,
4363        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4364        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4365    {
4366        let handler = Arc::new(handler);
4367        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4368            let state = state.downcast::<S>().map_err(|_| {
4369                "registered query state type does not match the replayed workflow state".to_string()
4370            })?;
4371            Ok(Box::pin(handler(ctx, state, args)))
4372        });
4373
4374        self.queries
4375            .entry(workflow_type.into())
4376            .or_default()
4377            .insert(
4378                query_name.into(),
4379                RegisteredQuery::Replayed {
4380                    state_type: TypeId::of::<S>(),
4381                    handler: erased_handler,
4382                },
4383            );
4384    }
4385
4386    /// Register a synchronous workflow update handler.
4387    pub fn register_update<F, Fut>(
4388        &mut self,
4389        workflow_type: impl Into<String>,
4390        update_name: impl Into<String>,
4391        handler: F,
4392    ) where
4393        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4394        Fut: Future<Output = Result<Value>> + Send + 'static,
4395    {
4396        let handler = Arc::new(handler);
4397        self.updates
4398            .entry(workflow_type.into())
4399            .or_default()
4400            .insert(
4401                update_name.into(),
4402                Arc::new(move |ctx, args| {
4403                    let handler = Arc::clone(&handler);
4404                    Box::pin(async move {
4405                        let result = handler(ctx, args.into_json()?).await?;
4406                        AvroValue::from_serialize(&result)
4407                    })
4408                }),
4409            );
4410    }
4411
4412    /// Register an update handler on the lossless fixed Avro Value surface.
4413    pub fn register_update_avro_value<F, Fut>(
4414        &mut self,
4415        workflow_type: impl Into<String>,
4416        update_name: impl Into<String>,
4417        handler: F,
4418    ) where
4419        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4420        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4421    {
4422        self.updates
4423            .entry(workflow_type.into())
4424            .or_default()
4425            .insert(
4426                update_name.into(),
4427                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4428            );
4429    }
4430
4431    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4432        let mut command_contracts = serde_json::Map::new();
4433        for workflow_type in self.workflows.keys() {
4434            let mut queries = self
4435                .queries
4436                .get(workflow_type)
4437                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4438                .unwrap_or_default();
4439            queries.sort();
4440            let mut updates = self
4441                .updates
4442                .get(workflow_type)
4443                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4444                .unwrap_or_default();
4445            updates.sort();
4446            if !queries.is_empty() || !updates.is_empty() {
4447                command_contracts.insert(
4448                    workflow_type.clone(),
4449                    json!({
4450                        "queries": queries,
4451                        "updates": updates,
4452                    }),
4453                );
4454            }
4455        }
4456
4457        self.client
4458            .register_worker_with_command_contracts(
4459                &self.worker_id,
4460                &self.task_queue,
4461                self.workflows.keys().cloned().collect(),
4462                self.activities.keys().cloned().collect(),
4463                self.max_concurrent_workflow_tasks,
4464                self.max_concurrent_activity_tasks,
4465                [
4466                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4467                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4468                ]
4469                .into_iter()
4470                .flatten()
4471                .collect(),
4472                Value::Object(command_contracts),
4473            )
4474            .await
4475    }
4476
4477    /// Run until shutdown or a terminal worker error occurs.
4478    ///
4479    /// Empty long-poll expirations do not stop the worker. Retryable poll and
4480    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
4481    /// authentication, protocol, and other non-retryable failures are returned.
4482    pub async fn run(&self) -> Result<()> {
4483        self.run_until(std::future::pending::<()>()).await
4484    }
4485
4486    /// Run until `shutdown` resolves or a terminal worker error occurs.
4487    ///
4488    /// This has the same liveness and terminal-error contract as [`Worker::run`].
4489    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4490    where
4491        F: Future<Output = ()>,
4492    {
4493        let registration = self.register().await?;
4494        let heartbeat_interval = Duration::from_secs(
4495            registration
4496                .heartbeat_interval_seconds
4497                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4498        );
4499        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
4500        // from the completion of the preceding attempt, including its bounded
4501        // retries. A fixed-epoch interval can leave an already-due tick queued
4502        // while an acknowledgement is slow, producing a catch-up heartbeat as
4503        // soon as that request completes.
4504        let heartbeat = tokio::time::sleep(Duration::ZERO);
4505        tokio::pin!(heartbeat);
4506        tokio::pin!(shutdown);
4507        let stop = Arc::new(AtomicBool::new(false));
4508        // Poll responses may already have leased server-side work by the time
4509        // they become ready, so each poller owns its responses through
4510        // completion or failure instead of racing raw polls in this select.
4511        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4512            let worker = self.clone();
4513            let stop = Arc::clone(&stop);
4514            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4515        });
4516        let mut activity_poller = (!self.activities.is_empty()).then(|| {
4517            let worker = self.clone();
4518            let stop = Arc::clone(&stop);
4519            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4520        });
4521        let mut query_poller = (!self.queries.is_empty()).then(|| {
4522            let worker = self.clone();
4523            let stop = Arc::clone(&stop);
4524            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4525        });
4526
4527        loop {
4528            tokio::select! {
4529                _ = &mut shutdown => {
4530                    stop.store(true, Ordering::SeqCst);
4531                    break;
4532                }
4533                _ = &mut heartbeat => {
4534                    let result = self.retry_worker_operation(|| {
4535                        self.client.heartbeat_worker(
4536                            &self.worker_id,
4537                            self.max_concurrent_workflow_tasks,
4538                            self.max_concurrent_activity_tasks,
4539                        )
4540                    }).await;
4541                    heartbeat
4542                        .as_mut()
4543                        .reset(tokio::time::Instant::now() + heartbeat_interval);
4544                    match result {
4545                        Ok(acknowledgement) => {
4546                            if let Some(observer) = &self.heartbeat_observer {
4547                                observer(&WorkerHeartbeatObservation {
4548                                    worker_id: self.worker_id.clone(),
4549                                    task_queue: self.task_queue.clone(),
4550                                    acknowledged_at_unix_millis: SystemTime::now()
4551                                        .duration_since(UNIX_EPOCH)
4552                                        .unwrap_or_default()
4553                                        .as_millis()
4554                                        .min(u64::MAX as u128)
4555                                        as u64,
4556                                    acknowledgement,
4557                                });
4558                            }
4559                        }
4560                        Err(error) => {
4561                            stop.store(true, Ordering::SeqCst);
4562                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4563                            return Err(error);
4564                        }
4565                    }
4566                }
4567                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4568                    workflow_poller = None;
4569                    let stopped_by_server = stop.load(Ordering::SeqCst);
4570                    stop.store(true, Ordering::SeqCst);
4571                    let poller_result = optional_poller_result("workflow", result);
4572                    let join_result =
4573                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4574                    poller_result?;
4575                    join_result?;
4576                    if stopped_by_server {
4577                        return Ok(());
4578                    }
4579                    return Err(Error::WorkerLoop(
4580                        "workflow poller stopped unexpectedly".to_string(),
4581                    ));
4582                }
4583                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4584                    activity_poller = None;
4585                    let stopped_by_server = stop.load(Ordering::SeqCst);
4586                    stop.store(true, Ordering::SeqCst);
4587                    let poller_result = optional_poller_result("activity", result);
4588                    let join_result =
4589                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4590                    poller_result?;
4591                    join_result?;
4592                    if stopped_by_server {
4593                        return Ok(());
4594                    }
4595                    return Err(Error::WorkerLoop(
4596                        "activity poller stopped unexpectedly".to_string(),
4597                    ));
4598                }
4599                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4600                    query_poller = None;
4601                    let stopped_by_server = stop.load(Ordering::SeqCst);
4602                    stop.store(true, Ordering::SeqCst);
4603                    let poller_result = optional_poller_result("query", result);
4604                    let join_result =
4605                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4606                    poller_result?;
4607                    join_result?;
4608                    if stopped_by_server {
4609                        return Ok(());
4610                    }
4611                    return Err(Error::WorkerLoop(
4612                        "query poller stopped unexpectedly".to_string(),
4613                    ));
4614                }
4615            }
4616        }
4617
4618        join_pollers(
4619            workflow_poller.take(),
4620            activity_poller.take(),
4621            query_poller.take(),
4622        )
4623        .await
4624    }
4625
4626    /// Poll and settle at most one task from each enabled task family.
4627    ///
4628    /// A workflow may reach its server-enforced run deadline while this worker
4629    /// holds a task. When the completion endpoint authoritatively rejects that
4630    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
4631    /// terminal `run_status=failed`, the workflow tick is considered settled:
4632    /// the late command was not recorded and cannot replace the terminal run.
4633    /// Every other completion rejection remains an error. This worker-level
4634    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
4635    /// only bounds how long a client waits for a result.
4636    ///
4637    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
4638    /// the original [`Error::Http`] status and response body.
4639    pub async fn run_once(&self) -> Result<usize> {
4640        let mut handled = 0;
4641        match self.poll_workflow_once().await? {
4642            ManagedPollOutcome::Handled => handled += 1,
4643            ManagedPollOutcome::Stop => return Ok(handled),
4644            ManagedPollOutcome::Idle => {}
4645        }
4646        match self.poll_activity_once().await? {
4647            ManagedPollOutcome::Handled => handled += 1,
4648            ManagedPollOutcome::Stop => return Ok(handled),
4649            ManagedPollOutcome::Idle => {}
4650        }
4651        if !self.queries.is_empty() {
4652            match self.poll_query_once().await? {
4653                ManagedPollOutcome::Handled => handled += 1,
4654                ManagedPollOutcome::Stop => return Ok(handled),
4655                ManagedPollOutcome::Idle => {}
4656            }
4657        }
4658        Ok(handled)
4659    }
4660
4661    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4662        let poll_request_id = unique_request_id("rust-workflow-poll");
4663        let response = self
4664            .retry_worker_operation(|| {
4665                self.client.poll_workflow_task_response_with_request_id(
4666                    &self.worker_id,
4667                    &self.task_queue,
4668                    self.poll_timeout,
4669                    &poll_request_id,
4670                    0,
4671                )
4672            })
4673            .await?;
4674        if response.outcome().should_stop() {
4675            return Ok(ManagedPollOutcome::Stop);
4676        }
4677        let Some(task) = response.task else {
4678            return Ok(ManagedPollOutcome::Idle);
4679        };
4680
4681        let task_id = task.task_id.clone();
4682        let attempt = task.workflow_task_attempt;
4683        let run_id = task.run_id.clone();
4684        let lease_owner = task
4685            .lease_owner
4686            .clone()
4687            .unwrap_or_else(|| self.worker_id.clone());
4688
4689        match self.execute_workflow_task(task) {
4690            Ok(commands) if commands.is_empty() => {
4691                // A replay can consume a recorded pending durable command
4692                // without producing a new command. The standalone protocol
4693                // acknowledges that state through the typed waiting outcome;
4694                // an empty completion is rejected by servers that require at
4695                // least one executable command.
4696                self.client
4697                    .fail_workflow_task_with_type(
4698                        &task_id,
4699                        &lease_owner,
4700                        attempt,
4701                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4702                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4703                    )
4704                    .await?;
4705            }
4706            Ok(commands) => {
4707                let completion = self
4708                    .client
4709                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4710                    .await;
4711                if let Err(error) = completion {
4712                    if !workflow_task_completion_is_terminal_timeout(
4713                        &error,
4714                        &task_id,
4715                        attempt,
4716                        run_id.as_deref(),
4717                    ) {
4718                        return Err(error);
4719                    }
4720                }
4721            }
4722            Err(error) => {
4723                self.client
4724                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4725                    .await?;
4726            }
4727        }
4728
4729        Ok(ManagedPollOutcome::Handled)
4730    }
4731
4732    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4733        while !stop.load(Ordering::SeqCst) {
4734            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4735                stop.store(true, Ordering::SeqCst);
4736                break;
4737            }
4738        }
4739
4740        Ok(())
4741    }
4742
4743    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4744        let poll_request_id = unique_request_id("rust-activity-poll");
4745        let response = self
4746            .retry_worker_operation(|| {
4747                self.client.poll_activity_task_response_with_request_id(
4748                    &self.worker_id,
4749                    &self.task_queue,
4750                    self.poll_timeout,
4751                    &poll_request_id,
4752                    0,
4753                )
4754            })
4755            .await?;
4756        if response.outcome().should_stop() {
4757            return Ok(ManagedPollOutcome::Stop);
4758        }
4759        let Some(task) = response.task else {
4760            return Ok(ManagedPollOutcome::Idle);
4761        };
4762
4763        let task_id = task.task_id.clone();
4764        let attempt_id = task
4765            .activity_attempt_id
4766            .clone()
4767            .or(task.attempt_id.clone())
4768            .unwrap_or_default();
4769        let lease_owner = task
4770            .lease_owner
4771            .clone()
4772            .unwrap_or_else(|| self.worker_id.clone());
4773        let codec = task.payload_codec.clone();
4774        let result = self.execute_activity_task(task).await;
4775        match result {
4776            Ok(value) => {
4777                let completion = self
4778                    .client
4779                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4780                    .await;
4781                if let Err(error) = completion {
4782                    if !activity_task_rejection_is_final(&error) {
4783                        return Err(error);
4784                    }
4785                }
4786            }
4787            Err(error) => {
4788                let failure = self
4789                    .client
4790                    .fail_activity_task(
4791                        &task_id,
4792                        &attempt_id,
4793                        &lease_owner,
4794                        error.to_string(),
4795                        false,
4796                    )
4797                    .await;
4798                if let Err(error) = failure {
4799                    if !activity_task_rejection_is_final(&error) {
4800                        return Err(error);
4801                    }
4802                }
4803            }
4804        }
4805
4806        Ok(ManagedPollOutcome::Handled)
4807    }
4808
4809    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4810        while !stop.load(Ordering::SeqCst) {
4811            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4812                stop.store(true, Ordering::SeqCst);
4813                break;
4814            }
4815        }
4816
4817        Ok(())
4818    }
4819
4820    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4821        let poll_request_id = unique_request_id("rust-query-poll");
4822        let response = self
4823            .retry_worker_operation(|| {
4824                self.client.poll_query_task_response_with_request_id(
4825                    &self.worker_id,
4826                    &self.task_queue,
4827                    self.poll_timeout,
4828                    &poll_request_id,
4829                    0,
4830                )
4831            })
4832            .await?;
4833        if response.outcome().should_stop() {
4834            return Ok(ManagedPollOutcome::Stop);
4835        }
4836        let Some(task) = response.task else {
4837            return Ok(ManagedPollOutcome::Idle);
4838        };
4839
4840        let query_task_id = task.query_task_id.clone();
4841        let attempt = task.query_task_attempt;
4842        let lease_owner = task
4843            .lease_owner
4844            .clone()
4845            .unwrap_or_else(|| self.worker_id.clone());
4846        let codec = task.payload_codec.clone();
4847
4848        match self.execute_query_task(task).await {
4849            Ok(value) => {
4850                let result_envelope = match encode_typed_envelope(&value, &codec) {
4851                    Ok(result_envelope) => result_envelope,
4852                    Err(error) => {
4853                        let failure = self
4854                            .client
4855                            .fail_query_task(
4856                                &query_task_id,
4857                                &lease_owner,
4858                                attempt,
4859                                error.to_string(),
4860                                "query_result_encode_failed",
4861                                "QueryResultEncodeFailed",
4862                            )
4863                            .await;
4864                        if let Err(error) = failure {
4865                            if !query_task_rejection_is_final(&error) {
4866                                return Err(error);
4867                            }
4868                        }
4869                        return Ok(ManagedPollOutcome::Handled);
4870                    }
4871                };
4872
4873                if let Err(error) = self
4874                    .client
4875                    .complete_query_task_with_envelope(
4876                        &query_task_id,
4877                        &lease_owner,
4878                        attempt,
4879                        value.clone().into_json()?,
4880                        result_envelope,
4881                    )
4882                    .await
4883                {
4884                    if !query_task_rejection_is_final(&error) {
4885                        return Err(error);
4886                    }
4887                }
4888            }
4889            Err(failure) => {
4890                let result = self
4891                    .client
4892                    .fail_query_task(
4893                        &query_task_id,
4894                        &lease_owner,
4895                        attempt,
4896                        failure.message,
4897                        failure.reason,
4898                        failure.failure_type,
4899                    )
4900                    .await;
4901                if let Err(error) = result {
4902                    if !query_task_rejection_is_final(&error) {
4903                        return Err(error);
4904                    }
4905                }
4906            }
4907        }
4908
4909        Ok(ManagedPollOutcome::Handled)
4910    }
4911
4912    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4913        while !stop.load(Ordering::SeqCst) {
4914            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
4915                stop.store(true, Ordering::SeqCst);
4916                break;
4917            }
4918        }
4919
4920        Ok(())
4921    }
4922
4923    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
4924    where
4925        F: FnMut() -> Fut,
4926        Fut: Future<Output = Result<T>>,
4927    {
4928        let mut retries = 0;
4929
4930        loop {
4931            match operation().await {
4932                Err(error)
4933                    if worker_operation_is_retryable(&error)
4934                        && retries < self.retry_policy.max_retries =>
4935                {
4936                    retries += 1;
4937                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
4938                }
4939                result => return result,
4940            }
4941        }
4942    }
4943
4944    async fn execute_query_task(
4945        &self,
4946        mut task: QueryTask,
4947    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
4948        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
4949            return Err(QueryTaskExecutionFailure::new(
4950                "query_payload_decode_failed",
4951                format!(
4952                    "cannot decode query payload with unsupported codec {:?}",
4953                    task.payload_codec
4954                ),
4955                "QueryPayloadDecodeFailed",
4956            ));
4957        }
4958
4959        if !self.workflows.contains_key(&task.workflow_type) {
4960            return Err(QueryTaskExecutionFailure::new(
4961                "query_workflow_type_not_registered",
4962                format!("no workflow registered for type {:?}", task.workflow_type),
4963                "WorkflowTypeNotRegistered",
4964            ));
4965        }
4966
4967        let Some(handlers) = self.queries.get(&task.workflow_type) else {
4968            return Err(QueryTaskExecutionFailure::new(
4969                "query_handler_unavailable",
4970                format!(
4971                    "query handlers are unavailable for workflow type {:?}",
4972                    task.workflow_type
4973                ),
4974                "QueryHandlerUnavailable",
4975            ));
4976        };
4977        let Some(query) = handlers.get(&task.query_name) else {
4978            return Err(QueryTaskExecutionFailure::new(
4979                "rejected_unknown_query",
4980                format!("unknown query {:?}", task.query_name),
4981                "QueryFailed",
4982            ));
4983        };
4984
4985        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
4986            .map_err(|error| {
4987            QueryTaskExecutionFailure::new(
4988                "query_payload_decode_failed",
4989                format!("cannot decode query arguments: {error}"),
4990                "QueryPayloadDecodeFailed",
4991            )
4992        })?;
4993        let workflow_input_typed =
4994            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
4995                .map_err(|error| {
4996                    QueryTaskExecutionFailure::new(
4997                        "query_workflow_state_unavailable",
4998                        format!("cannot decode workflow start input: {error}"),
4999                        "QueryWorkflowStateUnavailable",
5000                    )
5001                })?;
5002        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5003            QueryTaskExecutionFailure::new(
5004                "query_workflow_state_unavailable",
5005                format!("cannot project workflow start input: {error}"),
5006                "QueryWorkflowStateUnavailable",
5007            )
5008        })?;
5009        hydrate_query_history_from_export(&mut task).map_err(|error| {
5010            QueryTaskExecutionFailure::new(
5011                "query_workflow_state_unavailable",
5012                format!("cannot restore query history snapshot: {error}"),
5013                "QueryWorkflowStateUnavailable",
5014            )
5015        })?;
5016        enrich_query_history_from_export(&mut task).map_err(|error| {
5017            QueryTaskExecutionFailure::new(
5018                "query_workflow_state_unavailable",
5019                format!("cannot restore compact query history payloads: {error}"),
5020                "QueryWorkflowStateUnavailable",
5021            )
5022        })?;
5023        let signal_events = query_signal_events(&task).map_err(|error| {
5024            QueryTaskExecutionFailure::new(
5025                "query_workflow_state_unavailable",
5026                format!("cannot decode committed workflow signals: {error}"),
5027                "QueryWorkflowStateUnavailable",
5028            )
5029        })?;
5030        let history_events = Arc::new(std::mem::take(&mut task.history_events));
5031        let context = QueryContext {
5032            workflow_id: task.workflow_id,
5033            run_id: task.run_id,
5034            workflow_type: task.workflow_type.clone(),
5035            run_status: task.run_status,
5036            workflow_input,
5037            workflow_input_avro_value: workflow_input_typed.clone(),
5038            history_events: Arc::clone(&history_events),
5039            signal_events: Arc::new(signal_events),
5040        };
5041
5042        let future = match query {
5043            RegisteredQuery::Snapshot(handler) => handler(context, args),
5044            RegisteredQuery::Replayed {
5045                state_type,
5046                handler,
5047            } => {
5048                let workflow = self
5049                    .workflows
5050                    .get(&task.workflow_type)
5051                    .expect("workflow registration was checked above");
5052                if workflow.state_type != Some(*state_type) {
5053                    return Err(QueryTaskExecutionFailure::new(
5054                        "query_workflow_state_unavailable",
5055                        "replayed query state type does not match its workflow registration",
5056                        "QueryWorkflowStateUnavailable",
5057                    ));
5058                }
5059                let replay = workflow.replay.as_ref().ok_or_else(|| {
5060                    QueryTaskExecutionFailure::new(
5061                        "query_workflow_state_unavailable",
5062                        format!(
5063                            "workflow type {:?} is not registered for instance-state replay",
5064                            task.workflow_type
5065                        ),
5066                        "QueryWorkflowStateUnavailable",
5067                    )
5068                })?;
5069                let workflow_state = Arc::new(Mutex::new(
5070                    WorkflowState::new_with_identity(
5071                        history_events.as_ref().clone(),
5072                        context.workflow_id.clone(),
5073                        context.run_id.clone(),
5074                        self.task_queue.clone(),
5075                        task.payload_codec,
5076                        None,
5077                    )
5078                    .map_err(|error| {
5079                        QueryTaskExecutionFailure::new(
5080                            "query_workflow_state_unavailable",
5081                            format!("workflow replay failed before query: {error}"),
5082                            "QueryWorkflowStateUnavailable",
5083                        )
5084                    })?,
5085                ));
5086                let workflow_context = WorkflowContext {
5087                    state: workflow_state,
5088                };
5089                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5090                let mut cx = TaskContext::from_waker(noop_waker_ref());
5091                match invocation.future.as_mut().poll(&mut cx) {
5092                    Poll::Ready(Ok(_)) => {
5093                        workflow_context
5094                            .ensure_history_consumed()
5095                            .map_err(|error| {
5096                                QueryTaskExecutionFailure::new(
5097                                    "query_workflow_state_unavailable",
5098                                    format!("workflow replay failed before query: {error}"),
5099                                    "QueryWorkflowStateUnavailable",
5100                                )
5101                            })?;
5102                    }
5103                    Poll::Ready(Err(error)) => {
5104                        return Err(QueryTaskExecutionFailure::new(
5105                            "query_workflow_state_unavailable",
5106                            format!("workflow replay failed before query: {error}"),
5107                            "QueryWorkflowStateUnavailable",
5108                        ));
5109                    }
5110                    Poll::Pending => {
5111                        let commands = workflow_context.take_commands().map_err(|error| {
5112                            QueryTaskExecutionFailure::new(
5113                                "query_workflow_state_unavailable",
5114                                format!("workflow replay failed before query: {error}"),
5115                                "QueryWorkflowStateUnavailable",
5116                            )
5117                        })?;
5118                        if commands.is_empty()
5119                            && !workflow_context
5120                                .matched_recorded_pending()
5121                                .map_err(|error| {
5122                                    QueryTaskExecutionFailure::new(
5123                                        "query_workflow_state_unavailable",
5124                                        format!("workflow replay failed before query: {error}"),
5125                                        "QueryWorkflowStateUnavailable",
5126                                    )
5127                                })?
5128                        {
5129                            return Err(QueryTaskExecutionFailure::new(
5130                                "query_workflow_state_unavailable",
5131                                "workflow replay yielded without a durable command",
5132                                "QueryWorkflowStateUnavailable",
5133                            ));
5134                        }
5135                    }
5136                }
5137                let state = (invocation.snapshot)().map_err(|error| {
5138                    QueryTaskExecutionFailure::new(
5139                        "query_workflow_state_unavailable",
5140                        format!("cannot snapshot replayed workflow state: {error}"),
5141                        "QueryWorkflowStateUnavailable",
5142                    )
5143                })?;
5144                handler(context, state, args).map_err(|message| {
5145                    QueryTaskExecutionFailure::new(
5146                        "query_workflow_state_unavailable",
5147                        message,
5148                        "QueryWorkflowStateUnavailable",
5149                    )
5150                })?
5151            }
5152        };
5153
5154        future.await.map_err(|error| {
5155            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5156        })
5157    }
5158
5159    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5160        if let Some(update_id) = task
5161            .workflow_update_id
5162            .as_deref()
5163            .filter(|update_id| !update_id.is_empty())
5164        {
5165            return self.execute_update_task(&task, update_id);
5166        }
5167
5168        let workflow = self
5169            .workflows
5170            .get(&task.workflow_type)
5171            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5172        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5173        let resume_signal = decode_resume_signal(&task)?;
5174        let history_budget = WorkflowHistoryBudget {
5175            event_count: task
5176                .total_history_events
5177                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5178            size_bytes: task.history_size_bytes,
5179            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5180            pressure: task.history_budget_pressure.clone(),
5181        };
5182        let mut workflow_state = WorkflowState::new_with_identity(
5183            task.history_events,
5184            task.workflow_id,
5185            task.run_id,
5186            self.task_queue.clone(),
5187            task.payload_codec.clone(),
5188            resume_signal,
5189        )?;
5190        workflow_state.history_budget = history_budget;
5191        let state = Arc::new(Mutex::new(workflow_state));
5192        let ctx = WorkflowContext { state };
5193        let mut future = (workflow.execute)(ctx.clone(), input);
5194        let mut cx = TaskContext::from_waker(noop_waker_ref());
5195
5196        match future.as_mut().poll(&mut cx) {
5197            Poll::Ready(Ok(result)) => {
5198                ctx.ensure_history_consumed()?;
5199                let result = encode_typed_envelope(&result, &task.payload_codec)?;
5200                let mut commands = ctx.take_commands()?;
5201                commands.push(json!({
5202                    "type": "complete_workflow",
5203                    "result": result
5204                }));
5205                Ok(commands)
5206            }
5207            Poll::Ready(Err(error)) => {
5208                if let Error::ContinueAsNew(request) = error {
5209                    let mut commands = ctx.take_commands()?;
5210                    if let Some(command) = ctx.continue_as_new_command(request)? {
5211                        commands.push(command);
5212                    }
5213                    ctx.ensure_history_consumed()?;
5214                    return Ok(commands);
5215                }
5216                // A handler error must not hide a committed durable command that
5217                // upgraded workflow code no longer consumes.
5218                ctx.ensure_history_consumed()?;
5219                if workflow_task_integrity_error(&error) {
5220                    // Replay and protocol failures describe the workflow-task
5221                    // decision itself. Do not let commands queued earlier in
5222                    // this uncommitted decision escape alongside a terminal
5223                    // workflow failure.
5224                    return Err(error);
5225                }
5226                let mut commands = ctx.take_commands()?;
5227                commands.push(workflow_failure_command(&error));
5228                Ok(commands)
5229            }
5230            Poll::Pending => {
5231                let commands = ctx.take_commands()?;
5232                if commands.is_empty() && !ctx.matched_recorded_pending()? {
5233                    Err(Error::WorkflowYieldedWithoutCommand)
5234                } else {
5235                    Ok(commands)
5236                }
5237            }
5238        }
5239    }
5240
5241    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5242        if !self.workflows.contains_key(&task.workflow_type) {
5243            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5244        }
5245
5246        let accepted = task.history_events.iter().rev().find_map(|event| {
5247            (event.event_type == "UpdateAccepted"
5248                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5249            .then_some(&event.payload)
5250        });
5251        let update_name = accepted
5252            .and_then(|payload| payload.get("update_name"))
5253            .and_then(Value::as_str)
5254            .or(task.update_name.as_deref())
5255            .unwrap_or_default();
5256        let Some(handler) = self
5257            .updates
5258            .get(&task.workflow_type)
5259            .and_then(|handlers| handlers.get(update_name))
5260        else {
5261            return Ok(vec![json!({
5262                "type": "fail_update",
5263                "update_id": update_id,
5264                "message": format!(
5265                    "no update handler is registered for {}.{update_name}",
5266                    task.workflow_type
5267                ),
5268                "exception_type": "UnknownUpdate",
5269                "non_retryable": true,
5270            })]);
5271        };
5272        let arguments = accepted
5273            .and_then(|payload| payload.get("arguments"))
5274            .or(task.arguments.as_ref());
5275        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5276        let context = QueryContext {
5277            workflow_id: task.workflow_id.clone(),
5278            run_id: task.run_id.clone(),
5279            workflow_type: task.workflow_type.clone(),
5280            run_status: Some("running".to_string()),
5281            workflow_input: Value::Null,
5282            workflow_input_avro_value: AvroValue::Null,
5283            history_events: Arc::new(task.history_events.clone()),
5284            signal_events: Arc::new(Vec::new()),
5285        };
5286        let mut future = handler(context, arguments);
5287        let mut cx = TaskContext::from_waker(noop_waker_ref());
5288
5289        match future.as_mut().poll(&mut cx) {
5290            Poll::Ready(Ok(result)) => Ok(vec![json!({
5291                "type": "complete_update",
5292                "update_id": update_id,
5293                "result": encode_typed_envelope(&result, &task.payload_codec)?,
5294            })]),
5295            Poll::Ready(Err(error)) => Ok(vec![json!({
5296                "type": "fail_update",
5297                "update_id": update_id,
5298                "message": error.to_string(),
5299                "exception_type": "UpdateFailed",
5300                "non_retryable": true,
5301            })]),
5302            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5303        }
5304    }
5305
5306    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5307        let handler = self
5308            .activities
5309            .get(&task.activity_type)
5310            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5311        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5312        let attempt_id = task
5313            .activity_attempt_id
5314            .clone()
5315            .or(task.attempt_id.clone())
5316            .unwrap_or_default();
5317        let lease_owner = task
5318            .lease_owner
5319            .clone()
5320            .unwrap_or_else(|| self.worker_id.clone());
5321        let ctx = ActivityContext {
5322            client: self.client.clone(),
5323            task_id: task.task_id,
5324            activity_attempt_id: attempt_id,
5325            lease_owner,
5326            activity_type: task.activity_type,
5327            attempt_number: task.attempt_number,
5328            task_queue: self.task_queue.clone(),
5329            worker_id: self.worker_id.clone(),
5330        };
5331
5332        handler(ctx, args).await
5333    }
5334}
5335
5336fn poller_result(
5337    kind: &str,
5338    result: std::result::Result<Result<()>, tokio::task::JoinError>,
5339) -> Result<()> {
5340    match result {
5341        Ok(result) => result,
5342        Err(error) => Err(Error::WorkerLoop(format!(
5343            "{kind} poller join error: {error}"
5344        ))),
5345    }
5346}
5347
5348fn optional_poller_result(
5349    kind: &str,
5350    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5351) -> Result<()> {
5352    match result {
5353        Some(result) => poller_result(kind, result),
5354        None => Ok(()),
5355    }
5356}
5357
5358async fn join_pollers(
5359    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5360    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5361    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5362) -> Result<()> {
5363    let mut first_error = None;
5364
5365    if let Some(handle) = workflow_poller {
5366        if let Err(error) = poller_result("workflow", handle.await) {
5367            first_error.get_or_insert(error);
5368        }
5369    }
5370
5371    if let Some(handle) = activity_poller {
5372        if let Err(error) = poller_result("activity", handle.await) {
5373            first_error.get_or_insert(error);
5374        }
5375    }
5376
5377    if let Some(handle) = query_poller {
5378        if let Err(error) = poller_result("query", handle.await) {
5379            first_error.get_or_insert(error);
5380        }
5381    }
5382
5383    if let Some(error) = first_error {
5384        Err(error)
5385    } else {
5386        Ok(())
5387    }
5388}
5389
5390fn default_worker_id() -> String {
5391    let millis = SystemTime::now()
5392        .duration_since(UNIX_EPOCH)
5393        .unwrap_or_default()
5394        .as_millis();
5395    format!("rust-worker-{}-{millis}", std::process::id())
5396}
5397
5398fn unique_request_id(prefix: &str) -> String {
5399    let nanos = SystemTime::now()
5400        .duration_since(UNIX_EPOCH)
5401        .unwrap_or_default()
5402        .as_nanos();
5403    format!("{prefix}-{}-{nanos}", std::process::id())
5404}
5405
5406#[derive(Debug)]
5407struct QueryTaskExecutionFailure {
5408    reason: String,
5409    message: String,
5410    failure_type: String,
5411}
5412
5413impl QueryTaskExecutionFailure {
5414    fn new(
5415        reason: impl Into<String>,
5416        message: impl Into<String>,
5417        failure_type: impl Into<String>,
5418    ) -> Self {
5419        Self {
5420            reason: reason.into(),
5421            message: message.into(),
5422            failure_type: failure_type.into(),
5423        }
5424    }
5425}
5426
5427/// Typed local state owned by one deterministic workflow invocation.
5428///
5429/// Use [`WorkflowInstance::update`] for the same state transitions during
5430/// ordinary execution and replay. A replayed query receives a detached
5431/// immutable `Arc<S>` rather than this mutation-capable handle.
5432#[derive(Clone, Debug)]
5433pub struct WorkflowInstance<S> {
5434    state: Arc<Mutex<S>>,
5435}
5436
5437impl<S> WorkflowInstance<S> {
5438    fn new(state: S) -> Self {
5439        Self {
5440            state: Arc::new(Mutex::new(state)),
5441        }
5442    }
5443
5444    /// Read the current workflow-instance state without changing it.
5445    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5446        let state = self
5447            .state
5448            .lock()
5449            .map_err(|_| Error::WorkflowStatePoisoned)?;
5450        Ok(reader(&state))
5451    }
5452
5453    /// Apply one deterministic workflow-instance state transition.
5454    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5455        let mut state = self
5456            .state
5457            .lock()
5458            .map_err(|_| Error::WorkflowStatePoisoned)?;
5459        Ok(transition(&mut state))
5460    }
5461}
5462
5463impl<S: Clone> WorkflowInstance<S> {
5464    fn snapshot(&self) -> Result<S> {
5465        self.read(Clone::clone)
5466    }
5467}
5468
5469#[derive(Clone, Debug)]
5470pub struct WorkflowContext {
5471    state: Arc<Mutex<WorkflowState>>,
5472}
5473
5474impl WorkflowContext {
5475    /// Identity of the parent workflow currently being replayed.
5476    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5477        let state = self
5478            .state
5479            .lock()
5480            .map_err(|_| Error::WorkflowStatePoisoned)?;
5481        Ok(WorkflowIdentity {
5482            workflow_id: state.workflow_id.clone(),
5483            run_id: state.run_id.clone(),
5484        })
5485    }
5486
5487    /// Return the server-published history budget for this workflow task.
5488    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5489        let state = self
5490            .state
5491            .lock()
5492            .map_err(|_| Error::WorkflowStatePoisoned)?;
5493        Ok(state.history_budget.clone())
5494    }
5495
5496    /// Continue this workflow instance as a fresh run with replacement arguments.
5497    ///
5498    /// Return this value directly from the workflow handler. The worker converts
5499    /// it to the terminal protocol command only after replay has consumed every
5500    /// recorded durable command.
5501    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5502        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5503    }
5504
5505    /// Continue as new with optional workflow-type and task-queue overrides.
5506    pub fn continue_as_new_with_options<T: Serialize>(
5507        &self,
5508        options: ContinueAsNewOptions,
5509        args: T,
5510    ) -> Result<Value> {
5511        options.validate()?;
5512        Err(Error::ContinueAsNew(ContinueAsNewRequest {
5513            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5514            options,
5515        }))
5516    }
5517
5518    pub fn activity<T: Serialize>(
5519        &self,
5520        activity_type: impl Into<String>,
5521        args: T,
5522    ) -> ActivityCall {
5523        self.activity_with_options(activity_type, ActivityOptions::new(), args)
5524    }
5525
5526    pub fn activity_on_queue<T, Q>(
5527        &self,
5528        activity_type: impl Into<String>,
5529        task_queue: Option<Q>,
5530        args: T,
5531    ) -> ActivityCall
5532    where
5533        T: Serialize,
5534        Q: Into<String>,
5535    {
5536        let mut options = ActivityOptions::new();
5537        options.task_queue = task_queue.map(Into::into);
5538        self.activity_with_options(activity_type, options, args)
5539    }
5540
5541    /// Schedule one durable activity with retry, routing, and timeout options.
5542    ///
5543    /// Options are validated before the command is emitted. Once the command is
5544    /// recorded, replay consumes the same activity lifecycle at this command
5545    /// position and never emits a duplicate schedule.
5546    ///
5547    /// ```no_run
5548    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
5549    /// # use std::time::Duration;
5550    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
5551    /// let result = ctx
5552    ///     .activity_with_options(
5553    ///         "charge-card",
5554    ///         ActivityOptions::new()
5555    ///             .task_queue("payments")
5556    ///             .retry_policy(
5557    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
5558    ///                     Duration::from_secs(1),
5559    ///                     2,
5560    ///                     Some(Duration::from_secs(30)),
5561    ///                 ),
5562    ///             )
5563    ///             .start_to_close_timeout(Duration::from_secs(60))
5564    ///             .schedule_to_close_timeout(Duration::from_secs(180))
5565    ///             .heartbeat_timeout(Duration::from_secs(15)),
5566    ///         json!([{"order_id": "order-42"}]),
5567    ///     )
5568    ///     .await;
5569    /// match result {
5570    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
5571    ///         "reason": failure.reason,
5572    ///         "timeout_kind": failure.timeout_kind,
5573    ///     })),
5574    ///     other => other,
5575    /// }
5576    /// # }
5577    /// ```
5578    pub fn activity_with_options<T: Serialize>(
5579        &self,
5580        activity_type: impl Into<String>,
5581        options: ActivityOptions,
5582        args: T,
5583    ) -> ActivityCall {
5584        ActivityCall {
5585            ctx: self.clone(),
5586            activity_type: activity_type.into(),
5587            options,
5588            args: Some(AvroValue::from_serialize(&args)),
5589            scheduled: false,
5590        }
5591    }
5592
5593    pub async fn activity_avro_value<T: Serialize>(
5594        &self,
5595        activity_type: impl Into<String>,
5596        args: T,
5597    ) -> Result<AvroValue> {
5598        let mut call = self.activity(activity_type, args);
5599        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5600    }
5601
5602    pub async fn activity_avro_value_with_options<T: Serialize>(
5603        &self,
5604        activity_type: impl Into<String>,
5605        options: ActivityOptions,
5606        args: T,
5607    ) -> Result<AvroValue> {
5608        let mut call = self.activity_with_options(activity_type, options, args);
5609        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5610    }
5611
5612    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5613        SignalCall {
5614            ctx: self.clone(),
5615            signal_name: signal_name.into(),
5616            opened_wait: false,
5617            matched_pending: false,
5618        }
5619    }
5620
5621    pub async fn wait_signal_avro_value(
5622        &self,
5623        signal_name: impl Into<String>,
5624    ) -> Result<Vec<AvroValue>> {
5625        let mut call = self.wait_signal(signal_name);
5626        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5627    }
5628
5629    /// Wait for server-backed durable time without blocking the worker executor.
5630    ///
5631    /// Polling this future emits one `start_timer` command and yields. The
5632    /// server records the deadline, so neither worker nor server restarts reset
5633    /// the wait. Replay resolves the future only from a `TimerScheduled` and
5634    /// `TimerFired` pair at the same position in the shared durable-command
5635    /// stream, with matching sequence, timer identity, and delay. Sub-second
5636    /// durations round up because protocol deadlines use whole seconds.
5637    ///
5638    /// ```no_run
5639    /// # use durable_workflow::{json, Client, Worker};
5640    /// # use std::time::Duration;
5641    /// # fn configure(client: Client) {
5642    /// let mut worker = Worker::new(client, "rust-workers");
5643    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
5644    ///     ctx.sleep(Duration::from_secs(5)).await?;
5645    ///     Ok(json!({"status": "timer fired"}))
5646    /// });
5647    /// # }
5648    /// ```
5649    pub fn sleep(&self, duration: Duration) -> TimerCall {
5650        let delay_seconds = duration
5651            .as_secs()
5652            .checked_add(u64::from(duration.subsec_nanos() > 0));
5653        TimerCall {
5654            ctx: self.clone(),
5655            delay_seconds,
5656            scheduled: false,
5657            matched_pending: false,
5658        }
5659    }
5660
5661    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
5662    pub fn start_timer(&self, duration: Duration) -> TimerCall {
5663        self.sleep(duration)
5664    }
5665
5666    /// Evaluate a non-deterministic callback once and durably record its typed value.
5667    ///
5668    /// On replay the callback is not invoked: the value is decoded from the
5669    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
5670    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
5671    /// small values that must remain fixed for the lifetime of a workflow run.
5672    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5673    where
5674        T: Serialize + DeserializeOwned,
5675        F: FnOnce() -> T,
5676    {
5677        {
5678            let mut state = self
5679                .state
5680                .lock()
5681                .map_err(|_| Error::WorkflowStatePoisoned)?;
5682            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5683                return match recorded {
5684                    RecordedCommand::SideEffect { sequence, value } => {
5685                        state.command_cursor += 1;
5686                        value.deserialize().map_err(|error| {
5687                            Error::NonDeterministicReplay(ReplayFailure::new(
5688                                "side_effect_type_mismatch",
5689                                Some(sequence),
5690                                Some(std::any::type_name::<T>().to_string()),
5691                                Some(error.to_string()),
5692                                "recorded side-effect value is incompatible with the requested Rust type",
5693                            ))
5694                        })
5695                    }
5696                    other => Err(command_mismatch(&other, "side effect")),
5697                };
5698            }
5699        }
5700
5701        let value = callback();
5702        let avro_value = AvroValue::from_serialize(&value)?;
5703        let mut state = self
5704            .state
5705            .lock()
5706            .map_err(|_| Error::WorkflowStatePoisoned)?;
5707        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5708        state.commands.push(json!({
5709            "type": "record_side_effect",
5710            "result": result,
5711        }));
5712        Ok(value)
5713    }
5714
5715    /// Record or replay a lossless fixed Avro Value side effect.
5716    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5717    where
5718        F: FnOnce() -> AvroValue,
5719    {
5720        {
5721            let mut state = self
5722                .state
5723                .lock()
5724                .map_err(|_| Error::WorkflowStatePoisoned)?;
5725            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5726                return match recorded {
5727                    RecordedCommand::SideEffect { value, .. } => {
5728                        state.command_cursor += 1;
5729                        Ok(value)
5730                    }
5731                    other => Err(command_mismatch(&other, "side effect")),
5732                };
5733            }
5734        }
5735
5736        let value = callback();
5737        let mut state = self
5738            .state
5739            .lock()
5740            .map_err(|_| Error::WorkflowStatePoisoned)?;
5741        let result = encode_typed_envelope(&value, &state.payload_codec)?;
5742        state.commands.push(json!({
5743            "type": "record_side_effect",
5744            "result": result,
5745        }));
5746        Ok(value)
5747    }
5748
5749    /// Record a UUIDv4 once and return the same UUID on every replay.
5750    pub fn uuid_v4(&self) -> Result<Uuid> {
5751        self.side_effect(Uuid::new_v4)
5752    }
5753
5754    /// Select the newest supported version for a change, or replay the version
5755    /// already committed for that stable change ID.
5756    pub fn get_version(
5757        &self,
5758        change_id: impl Into<String>,
5759        min_supported: i32,
5760        max_supported: i32,
5761    ) -> Result<i32> {
5762        let change_id = change_id.into();
5763        if change_id.trim().is_empty() {
5764            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5765                "version_change_id_invalid",
5766                None,
5767                Some("non-empty change ID".to_string()),
5768                Some(change_id),
5769                "version markers require a stable non-empty change ID",
5770            )));
5771        }
5772        if min_supported > max_supported {
5773            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5774                "version_range_invalid",
5775                None,
5776                Some("min_supported <= max_supported".to_string()),
5777                Some(format!("{min_supported}..={max_supported}")),
5778                "version marker supported range is invalid",
5779            )));
5780        }
5781
5782        let mut state = self
5783            .state
5784            .lock()
5785            .map_err(|_| Error::WorkflowStatePoisoned)?;
5786        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5787            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5788            return Ok(version);
5789        }
5790
5791        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5792            return match recorded {
5793                RecordedCommand::VersionMarker {
5794                    sequence,
5795                    change_id: recorded_change_id,
5796                    version,
5797                    ..
5798                } => {
5799                    if recorded_change_id != change_id {
5800                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5801                            "version_change_id_mismatch",
5802                            Some(sequence),
5803                            Some(recorded_change_id),
5804                            Some(change_id),
5805                            "recorded version marker change ID differs from current workflow code",
5806                        )));
5807                    }
5808                    ensure_version_supported(
5809                        &change_id,
5810                        version,
5811                        min_supported,
5812                        max_supported,
5813                        sequence,
5814                    )?;
5815                    state.command_cursor += 1;
5816                    state.version_markers.insert(change_id, (version, sequence));
5817                    Ok(version)
5818                }
5819                other => Err(command_mismatch(
5820                    &other,
5821                    format!("version marker:{change_id}"),
5822                )),
5823            };
5824        }
5825
5826        let version = max_supported;
5827        state.commands.push(json!({
5828            "type": "record_version_marker",
5829            "change_id": change_id,
5830            "version": version,
5831            "min_supported": min_supported,
5832            "max_supported": max_supported,
5833        }));
5834        // Sequence numbers are assigned by the server. Zero identifies a marker
5835        // selected in this uncommitted decision batch for duplicate-call checks.
5836        state.version_markers.insert(change_id, (version, 0));
5837        Ok(version)
5838    }
5839
5840    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
5841    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5842        Ok(self.get_version(change_id, -1, 1)? == 1)
5843    }
5844
5845    /// Keep a patch marker in history after the legacy branch has been removed.
5846    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5847        self.get_version(change_id, -1, 1).map(|_| ())
5848    }
5849
5850    /// Start a named durable child on an explicit queue and await its result.
5851    ///
5852    /// The command is recorded in the parent's sequence-ordered durable command
5853    /// stream. Replay keeps a scheduled child pending without emitting another
5854    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
5855    /// values preserve the history payload codec and include both sides of the
5856    /// durable relationship; failures are returned as
5857    /// [`Error::ChildWorkflowFailed`].
5858    ///
5859    /// ```no_run
5860    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
5861    /// # fn configure(client: Client) {
5862    /// let mut worker = Worker::new(client, "parent-workers");
5863    /// worker.register_workflow("order-parent", |ctx, _input| async move {
5864    ///     let child = ctx
5865    ///         .start_child_workflow(
5866    ///             "fulfil-order",
5867    ///             ChildWorkflowOptions::new("fulfilment-workers")
5868    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
5869    ///             json!([{"order_id": "order-42"}]),
5870    ///         )
5871    ///         .await?;
5872    ///     Ok(child.result)
5873    /// });
5874    /// # }
5875    /// ```
5876    pub fn start_child_workflow<T: Serialize>(
5877        &self,
5878        workflow_type: impl Into<String>,
5879        options: ChildWorkflowOptions,
5880        args: T,
5881    ) -> ChildWorkflowCall {
5882        ChildWorkflowCall {
5883            ctx: self.clone(),
5884            workflow_type: workflow_type.into(),
5885            options,
5886            args: Some(AvroValue::from_serialize(&args)),
5887            scheduled: false,
5888            matched_pending: false,
5889        }
5890    }
5891
5892    pub async fn start_child_workflow_avro_value<T: Serialize>(
5893        &self,
5894        workflow_type: impl Into<String>,
5895        options: ChildWorkflowOptions,
5896        args: T,
5897    ) -> Result<ChildWorkflowAvroResult> {
5898        let mut call = self.start_child_workflow(workflow_type, options, args);
5899        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5900    }
5901
5902    fn take_commands(&self) -> Result<Vec<Value>> {
5903        let mut state = self
5904            .state
5905            .lock()
5906            .map_err(|_| Error::WorkflowStatePoisoned)?;
5907        Ok(std::mem::take(&mut state.commands))
5908    }
5909
5910    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
5911        let mut state = self
5912            .state
5913            .lock()
5914            .map_err(|_| Error::WorkflowStatePoisoned)?;
5915
5916        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5917            return Err(command_mismatch(&recorded, "continue as new"));
5918        }
5919        if state.recorded_continue_as_new_sequence.is_some() {
5920            state.continue_as_new_consumed = true;
5921            return Ok(None);
5922        }
5923
5924        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
5925        let mut command = serde_json::Map::from_iter([
5926            ("type".to_string(), json!("continue_as_new")),
5927            ("arguments".to_string(), arguments),
5928            ("queue".to_string(), json!(state.task_queue.clone())),
5929        ]);
5930        if let Some(workflow_type) = request.options.workflow_type {
5931            command.insert("workflow_type".to_string(), json!(workflow_type));
5932        }
5933        if let Some(task_queue) = request.options.task_queue {
5934            command.insert("queue".to_string(), json!(task_queue));
5935        }
5936        Ok(Some(Value::Object(command)))
5937    }
5938
5939    fn matched_recorded_pending(&self) -> Result<bool> {
5940        let state = self
5941            .state
5942            .lock()
5943            .map_err(|_| Error::WorkflowStatePoisoned)?;
5944        Ok(state.matched_recorded_pending)
5945    }
5946
5947    fn ensure_history_consumed(&self) -> Result<()> {
5948        let state = self
5949            .state
5950            .lock()
5951            .map_err(|_| Error::WorkflowStatePoisoned)?;
5952        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
5953            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5954                "recorded_commands_unconsumed",
5955                Some(command.sequence()),
5956                Some(command.shape().to_string()),
5957                Some("workflow completion".to_string()),
5958                "workflow completed before consuming all recorded durable commands",
5959            )));
5960        }
5961        if let Some(sequence) = state
5962            .recorded_continue_as_new_sequence
5963            .filter(|_| !state.continue_as_new_consumed)
5964        {
5965            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5966                "recorded_continue_as_new_unconsumed",
5967                Some(sequence),
5968                Some("continue as new".to_string()),
5969                Some("workflow completion".to_string()),
5970                "workflow completed without consuming its recorded continue-as-new transition",
5971            )));
5972        }
5973        Ok(())
5974    }
5975}
5976
5977#[derive(Debug)]
5978struct WorkflowState {
5979    workflow_id: Option<String>,
5980    run_id: Option<String>,
5981    task_queue: String,
5982    payload_codec: String,
5983    history_budget: WorkflowHistoryBudget,
5984    resume_signal: Option<ResumeSignal>,
5985    recorded_commands: Vec<RecordedCommand>,
5986    recorded_continue_as_new_sequence: Option<u64>,
5987    continue_as_new_consumed: bool,
5988    command_cursor: usize,
5989    matched_recorded_pending: bool,
5990    version_markers: HashMap<String, (i32, u64)>,
5991    commands: Vec<Value>,
5992}
5993
5994impl WorkflowState {
5995    #[cfg(test)]
5996    fn new(
5997        history: Vec<HistoryEvent>,
5998        task_queue: String,
5999        payload_codec: String,
6000        resume_signal: Option<ResumeSignal>,
6001    ) -> Result<Self> {
6002        Self::new_with_identity(
6003            history,
6004            None,
6005            None,
6006            task_queue,
6007            payload_codec,
6008            resume_signal,
6009        )
6010    }
6011
6012    fn new_with_identity(
6013        history: Vec<HistoryEvent>,
6014        workflow_id: Option<String>,
6015        run_id: Option<String>,
6016        task_queue: String,
6017        payload_codec: String,
6018        resume_signal: Option<ResumeSignal>,
6019    ) -> Result<Self> {
6020        let recorded_commands = recorded_commands(
6021            &history,
6022            &payload_codec,
6023            WorkflowIdentity {
6024                workflow_id: workflow_id.clone(),
6025                run_id: run_id.clone(),
6026            },
6027        )?;
6028        let recorded_continue_as_new = history
6029            .iter()
6030            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6031            .collect::<Vec<_>>();
6032        if recorded_continue_as_new.len() > 1 {
6033            return Err(invalid_recorded_history(
6034                "duplicate_continue_as_new_transition",
6035                recorded_continue_as_new
6036                    .last()
6037                    .and_then(|event| durable_event_sequence(event))
6038                    .unwrap_or(0),
6039                "one WorkflowContinuedAsNew event",
6040                &format!(
6041                    "{} WorkflowContinuedAsNew events",
6042                    recorded_continue_as_new.len()
6043                ),
6044                "workflow history records one continue-as-new transition more than once",
6045            ));
6046        }
6047        let recorded_continue_as_new_sequence = recorded_continue_as_new
6048            .first()
6049            .map(|event| {
6050                durable_event_sequence(event).ok_or_else(|| {
6051                    Error::NonDeterministicReplay(ReplayFailure::new(
6052                        "continue_as_new_sequence_missing",
6053                        None,
6054                        Some("recorded transition sequence".to_string()),
6055                        Some("missing sequence".to_string()),
6056                        "WorkflowContinuedAsNew history is missing its recorded sequence",
6057                    ))
6058                })
6059            })
6060            .transpose()?;
6061        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6062        Ok(Self {
6063            workflow_id,
6064            run_id,
6065            task_queue,
6066            payload_codec,
6067            history_budget: WorkflowHistoryBudget {
6068                event_count,
6069                ..WorkflowHistoryBudget::default()
6070            },
6071            resume_signal,
6072            recorded_commands,
6073            recorded_continue_as_new_sequence,
6074            continue_as_new_consumed: false,
6075            command_cursor: 0,
6076            matched_recorded_pending: false,
6077            version_markers: HashMap::new(),
6078            commands: Vec::new(),
6079        })
6080    }
6081}
6082
6083#[derive(Clone, Debug)]
6084enum RecordedCommand {
6085    Activity {
6086        sequence: u64,
6087        activity_type: Option<String>,
6088        options: Option<RecordedActivityOptions>,
6089        outcome: Option<ActivityOutcome>,
6090    },
6091    Timer {
6092        sequence: u64,
6093        delay_seconds: u64,
6094        fired: bool,
6095    },
6096    ChildWorkflow {
6097        sequence: u64,
6098        workflow_type: Option<String>,
6099        outcome: Option<ChildWorkflowOutcome>,
6100    },
6101    SignalWait {
6102        sequence: u64,
6103        signal_name: String,
6104        value: Option<Vec<AvroValue>>,
6105    },
6106    SideEffect {
6107        sequence: u64,
6108        value: AvroValue,
6109    },
6110    VersionMarker {
6111        sequence: u64,
6112        change_id: String,
6113        version: i32,
6114    },
6115}
6116
6117#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6118struct RecordedActivityOptions {
6119    task_queue: RecordedSnapshotValue<Option<String>>,
6120    execution_mode: RecordedSnapshotValue<Option<String>>,
6121    retry_policy: ActivityRetrySnapshot,
6122}
6123
6124#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6125enum RecordedSnapshotValue<T> {
6126    /// Older history did not persist this field, so it cannot constrain replay.
6127    Unknown,
6128    Known(T),
6129}
6130
6131impl<T: PartialEq> RecordedSnapshotValue<T> {
6132    fn matches_current(&self, current: &Self) -> bool {
6133        match self {
6134            Self::Unknown => true,
6135            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6136        }
6137    }
6138}
6139
6140#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6141struct ActivityRetrySnapshot {
6142    snapshot_version: RecordedSnapshotValue<Option<u64>>,
6143    max_attempts: RecordedSnapshotValue<Option<u64>>,
6144    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6145    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6146    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6147    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6148    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6149    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6150}
6151
6152impl ActivityRetrySnapshot {
6153    fn matches_current(&self, current: &Self) -> bool {
6154        self.snapshot_version
6155            .matches_current(&current.snapshot_version)
6156            && self.max_attempts.matches_current(&current.max_attempts)
6157            && self
6158                .backoff_seconds
6159                .matches_current(&current.backoff_seconds)
6160            && self
6161                .start_to_close_timeout
6162                .matches_current(&current.start_to_close_timeout)
6163            && self
6164                .schedule_to_start_timeout
6165                .matches_current(&current.schedule_to_start_timeout)
6166            && self
6167                .schedule_to_close_timeout
6168                .matches_current(&current.schedule_to_close_timeout)
6169            && self
6170                .heartbeat_timeout
6171                .matches_current(&current.heartbeat_timeout)
6172            && self
6173                .non_retryable_error_types
6174                .matches_current(&current.non_retryable_error_types)
6175    }
6176}
6177
6178fn recorded_optional_u64(
6179    object: Option<&serde_json::Map<String, Value>>,
6180    field: &str,
6181) -> RecordedSnapshotValue<Option<u64>> {
6182    match object.and_then(|object| object.get(field)) {
6183        None => RecordedSnapshotValue::Unknown,
6184        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6185        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6186    }
6187}
6188
6189fn recorded_optional_string(
6190    object: &serde_json::Map<String, Value>,
6191    field: &str,
6192) -> RecordedSnapshotValue<Option<String>> {
6193    match object.get(field) {
6194        None => RecordedSnapshotValue::Unknown,
6195        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6196        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6197    }
6198}
6199
6200fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6201    let policy = policy.and_then(Value::as_object);
6202    let backoff_seconds = policy
6203        .and_then(|policy| policy.get("backoff_seconds"))
6204        .and_then(Value::as_array)
6205        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6206        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6207    let mut non_retryable_error_types = Vec::new();
6208    for error_type in policy
6209        .and_then(|policy| policy.get("non_retryable_error_types"))
6210        .and_then(Value::as_array)
6211        .into_iter()
6212        .flatten()
6213        .filter_map(Value::as_str)
6214        .map(str::trim)
6215        .filter(|error_type| !error_type.is_empty())
6216    {
6217        if !non_retryable_error_types
6218            .iter()
6219            .any(|recorded| recorded == error_type)
6220        {
6221            non_retryable_error_types.push(error_type.to_string());
6222        }
6223    }
6224
6225    ActivityRetrySnapshot {
6226        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6227        max_attempts: recorded_optional_u64(policy, "max_attempts"),
6228        backoff_seconds,
6229        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6230        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6231        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6232        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6233        non_retryable_error_types: if policy
6234            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6235        {
6236            RecordedSnapshotValue::Known(non_retryable_error_types)
6237        } else {
6238            RecordedSnapshotValue::Unknown
6239        },
6240    }
6241}
6242
6243fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6244    let policy = options.retry_policy.as_ref();
6245    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6246        Some(Value::Null) => None,
6247        Some(value) => value_as_u64(value),
6248        None => Some(1),
6249    };
6250    let backoff_seconds = policy
6251        .and_then(|policy| policy.get("backoff_seconds"))
6252        .and_then(Value::as_array)
6253        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6254        .unwrap_or_default();
6255    let non_retryable_error_types = policy
6256        .and_then(|policy| policy.get("non_retryable_error_types"))
6257        .and_then(Value::as_array)
6258        .into_iter()
6259        .flatten()
6260        .filter_map(Value::as_str)
6261        .map(str::to_string)
6262        .collect();
6263
6264    ActivityRetrySnapshot {
6265        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6266        max_attempts: RecordedSnapshotValue::Known(max_attempts),
6267        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6268        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6269        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6270        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6271        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6272        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6273    }
6274}
6275
6276fn activity_options_description(options: &RecordedActivityOptions) -> String {
6277    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6278}
6279
6280impl RecordedCommand {
6281    fn sequence(&self) -> u64 {
6282        match self {
6283            Self::Activity { sequence, .. }
6284            | Self::Timer { sequence, .. }
6285            | Self::ChildWorkflow { sequence, .. }
6286            | Self::SignalWait { sequence, .. }
6287            | Self::SideEffect { sequence, .. }
6288            | Self::VersionMarker { sequence, .. } => *sequence,
6289        }
6290    }
6291
6292    fn shape(&self) -> &'static str {
6293        match self {
6294            Self::Activity { .. } => "activity",
6295            Self::Timer { .. } => "timer",
6296            Self::ChildWorkflow { .. } => "child workflow",
6297            Self::SignalWait { .. } => "signal wait",
6298            Self::SideEffect { .. } => "side effect",
6299            Self::VersionMarker { .. } => "version marker",
6300        }
6301    }
6302}
6303
6304fn ensure_version_supported(
6305    change_id: &str,
6306    version: i32,
6307    min_supported: i32,
6308    max_supported: i32,
6309    sequence: u64,
6310) -> Result<()> {
6311    if (min_supported..=max_supported).contains(&version) {
6312        return Ok(());
6313    }
6314    Err(Error::NonDeterministicReplay(ReplayFailure::new(
6315        "version_marker_incompatible_range",
6316        (sequence != 0).then_some(sequence),
6317        Some(format!("{min_supported}..={max_supported}")),
6318        Some(format!("{change_id}:{version}")),
6319        "recorded workflow version is outside the range supported by current code",
6320    )))
6321}
6322
6323#[derive(Clone, Debug)]
6324struct ResumeSignal {
6325    signal_name: String,
6326    arguments: Vec<AvroValue>,
6327}
6328
6329pub struct ActivityCall {
6330    ctx: WorkflowContext,
6331    activity_type: String,
6332    options: ActivityOptions,
6333    args: Option<Result<AvroValue>>,
6334    scheduled: bool,
6335}
6336
6337impl ActivityCall {
6338    fn poll_avro_value(
6339        mut self: Pin<&mut Self>,
6340        _cx: &mut TaskContext<'_>,
6341    ) -> Poll<Result<AvroValue>> {
6342        let ctx = self.ctx.clone();
6343        let mut state = match ctx.state.lock() {
6344            Ok(state) => state,
6345            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6346        };
6347
6348        if self.scheduled {
6349            return Poll::Pending;
6350        }
6351
6352        let options = match self.options.validate() {
6353            Ok(options) => options,
6354            Err(error) => {
6355                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6356            }
6357        };
6358        let task_queue = options
6359            .task_queue
6360            .clone()
6361            .unwrap_or_else(|| state.task_queue.clone());
6362        let current_recorded_options = RecordedActivityOptions {
6363            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6364            // Rust schedules ordinary durable activities. The server records a
6365            // non-null mode only for a specialized execution primitive.
6366            execution_mode: RecordedSnapshotValue::Known(None),
6367            retry_policy: current_activity_retry_snapshot(&options),
6368        };
6369
6370        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6371            let sequence = recorded.sequence();
6372            match recorded {
6373                RecordedCommand::Activity {
6374                    activity_type,
6375                    options: recorded_options,
6376                    outcome,
6377                    ..
6378                } => {
6379                    if let Some(recorded_type) = activity_type {
6380                        if recorded_type != self.activity_type {
6381                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6382                                ReplayFailure::new(
6383                                    "recorded_command_detail_mismatch",
6384                                    Some(sequence),
6385                                    Some(format!("activity:{recorded_type}")),
6386                                    Some(format!("activity:{}", self.activity_type)),
6387                                    "recorded activity type differs from the current workflow command",
6388                                ),
6389                            )));
6390                        }
6391                    }
6392                    if let Some(recorded_options) = recorded_options {
6393                        if !recorded_options
6394                            .task_queue
6395                            .matches_current(&current_recorded_options.task_queue)
6396                        {
6397                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6398                                ReplayFailure::new(
6399                                    "activity_task_queue_mismatch",
6400                                    Some(sequence),
6401                                    Some(activity_options_description(&recorded_options)),
6402                                    Some(activity_options_description(&current_recorded_options)),
6403                                    "recorded activity task queue differs from the current workflow command",
6404                                ),
6405                            )));
6406                        }
6407                        if !recorded_options
6408                            .execution_mode
6409                            .matches_current(&current_recorded_options.execution_mode)
6410                        {
6411                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6412                                ReplayFailure::new(
6413                                    "activity_execution_mode_mismatch",
6414                                    Some(sequence),
6415                                    Some(activity_options_description(&recorded_options)),
6416                                    Some(activity_options_description(&current_recorded_options)),
6417                                    "recorded activity execution mode differs from the current workflow command",
6418                                ),
6419                            )));
6420                        }
6421                        if !recorded_options
6422                            .retry_policy
6423                            .matches_current(&current_recorded_options.retry_policy)
6424                        {
6425                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6426                                ReplayFailure::new(
6427                                    "activity_retry_policy_mismatch",
6428                                    Some(sequence),
6429                                    Some(activity_options_description(&recorded_options)),
6430                                    Some(activity_options_description(&current_recorded_options)),
6431                                    "recorded activity retry policy differs from the current workflow command",
6432                                ),
6433                            )));
6434                        }
6435                    }
6436                    state.command_cursor += 1;
6437                    if let Some(outcome) = outcome {
6438                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6439                    }
6440                    state.matched_recorded_pending = true;
6441                    self.scheduled = true;
6442                    return Poll::Pending;
6443                }
6444                other => {
6445                    return Poll::Ready(Err(command_mismatch(
6446                        &other,
6447                        format!("activity:{}", self.activity_type),
6448                    )));
6449                }
6450            }
6451        }
6452
6453        if !self.scheduled {
6454            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6455                Ok(args) => args,
6456                Err(error) => return Poll::Ready(Err(error)),
6457            };
6458            let arguments = normalize_avro_arguments(args);
6459            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6460                Ok(envelope) => envelope,
6461                Err(error) => return Poll::Ready(Err(error)),
6462            };
6463
6464            let mut command = serde_json::Map::from_iter([
6465                ("type".to_string(), json!("schedule_activity")),
6466                (
6467                    "activity_type".to_string(),
6468                    json!(self.activity_type.clone()),
6469                ),
6470                ("queue".to_string(), json!(task_queue)),
6471                ("arguments".to_string(), envelope),
6472            ]);
6473            for (field, value) in [
6474                ("start_to_close_timeout", options.start_to_close_timeout),
6475                (
6476                    "schedule_to_start_timeout",
6477                    options.schedule_to_start_timeout,
6478                ),
6479                (
6480                    "schedule_to_close_timeout",
6481                    options.schedule_to_close_timeout,
6482                ),
6483                ("heartbeat_timeout", options.heartbeat_timeout),
6484            ] {
6485                if let Some(value) = value {
6486                    command.insert(field.to_string(), json!(value));
6487                }
6488            }
6489            if let Some(retry_policy) = options.retry_policy {
6490                command.insert("retry_policy".to_string(), retry_policy);
6491            }
6492            state.commands.push(Value::Object(command));
6493            self.scheduled = true;
6494        }
6495
6496        Poll::Pending
6497    }
6498}
6499
6500impl Future for ActivityCall {
6501    type Output = Result<Value>;
6502
6503    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6504        match self.poll_avro_value(cx) {
6505            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6506            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6507            Poll::Pending => Poll::Pending,
6508        }
6509    }
6510}
6511
6512/// Future returned by [`WorkflowContext::sleep`].
6513pub struct TimerCall {
6514    ctx: WorkflowContext,
6515    delay_seconds: Option<u64>,
6516    scheduled: bool,
6517    matched_pending: bool,
6518}
6519
6520impl Future for TimerCall {
6521    type Output = Result<()>;
6522
6523    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6524        if self.matched_pending {
6525            return Poll::Pending;
6526        }
6527
6528        let ctx = self.ctx.clone();
6529        let Some(requested_delay) = self.delay_seconds else {
6530            return Poll::Ready(Err(Error::TimerDurationOverflow));
6531        };
6532        let mut state = match ctx.state.lock() {
6533            Ok(state) => state,
6534            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6535        };
6536
6537        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6538            match recorded {
6539                RecordedCommand::Timer {
6540                    sequence,
6541                    delay_seconds,
6542                    fired,
6543                    ..
6544                } => {
6545                    if delay_seconds != requested_delay {
6546                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6547                            ReplayFailure::new(
6548                                "timer_delay_mismatch",
6549                                Some(sequence),
6550                                Some(format!("timer:{delay_seconds}s")),
6551                                Some(format!("timer:{requested_delay}s")),
6552                                "recorded timer delay differs from the current workflow command",
6553                            ),
6554                        )));
6555                    }
6556                    state.command_cursor += 1;
6557                    if fired {
6558                        return Poll::Ready(Ok(()));
6559                    }
6560                    state.matched_recorded_pending = true;
6561                    self.scheduled = true;
6562                    self.matched_pending = true;
6563                    return Poll::Pending;
6564                }
6565                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6566            }
6567        }
6568
6569        if !self.scheduled {
6570            state.commands.push(json!({
6571                "type": "start_timer",
6572                "delay_seconds": requested_delay,
6573            }));
6574            self.scheduled = true;
6575        }
6576
6577        Poll::Pending
6578    }
6579}
6580
6581/// Future returned by [`WorkflowContext::start_child_workflow`].
6582pub struct ChildWorkflowCall {
6583    ctx: WorkflowContext,
6584    workflow_type: String,
6585    options: ChildWorkflowOptions,
6586    args: Option<Result<AvroValue>>,
6587    scheduled: bool,
6588    matched_pending: bool,
6589}
6590
6591impl ChildWorkflowCall {
6592    fn poll_avro_value(
6593        mut self: Pin<&mut Self>,
6594        _cx: &mut TaskContext<'_>,
6595    ) -> Poll<Result<ChildWorkflowAvroResult>> {
6596        if self.matched_pending {
6597            return Poll::Pending;
6598        }
6599
6600        let ctx = self.ctx.clone();
6601        let mut state = match ctx.state.lock() {
6602            Ok(state) => state,
6603            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6604        };
6605
6606        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6607            let sequence = recorded.sequence();
6608            match recorded {
6609                RecordedCommand::ChildWorkflow {
6610                    workflow_type,
6611                    outcome,
6612                    ..
6613                } => {
6614                    if let Some(recorded_type) = workflow_type {
6615                        if recorded_type != self.workflow_type {
6616                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6617                                ReplayFailure::new(
6618                                    "recorded_command_detail_mismatch",
6619                                    Some(sequence),
6620                                    Some(format!("child workflow:{recorded_type}")),
6621                                    Some(format!("child workflow:{}", self.workflow_type)),
6622                                    "recorded child workflow type differs from the current workflow command",
6623                                ),
6624                            )));
6625                        }
6626                    }
6627                    state.command_cursor += 1;
6628                    if let Some(outcome) = outcome {
6629                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6630                    }
6631                    state.matched_recorded_pending = true;
6632                    self.scheduled = true;
6633                    self.matched_pending = true;
6634                    return Poll::Pending;
6635                }
6636                other => {
6637                    return Poll::Ready(Err(command_mismatch(
6638                        &other,
6639                        format!("child workflow:{}", self.workflow_type),
6640                    )));
6641                }
6642            }
6643        }
6644
6645        if !self.scheduled {
6646            if self.options.task_queue.trim().is_empty() {
6647                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6648                    "task_queue must not be empty".to_string(),
6649                )));
6650            }
6651            for (name, value) in [
6652                (
6653                    "execution_timeout_seconds",
6654                    self.options.execution_timeout_seconds,
6655                ),
6656                ("run_timeout_seconds", self.options.run_timeout_seconds),
6657            ] {
6658                if value == Some(0) {
6659                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6660                        "{name} must be at least 1"
6661                    ))));
6662                }
6663            }
6664
6665            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6666                Ok(args) => args,
6667                Err(error) => return Poll::Ready(Err(error)),
6668            };
6669            let arguments = match encode_typed_envelope(
6670                &normalize_avro_arguments(args),
6671                &state.payload_codec,
6672            ) {
6673                Ok(arguments) => arguments,
6674                Err(error) => return Poll::Ready(Err(error)),
6675            };
6676            let mut command = json!({
6677                "type": "start_child_workflow",
6678                "workflow_type": self.workflow_type,
6679                "queue": self.options.task_queue,
6680                "parent_close_policy": self.options.parent_close_policy.as_str(),
6681                "arguments": arguments,
6682            });
6683            let object = command
6684                .as_object_mut()
6685                .expect("child workflow command is always an object");
6686            if let Some(policy) = &self.options.retry_policy {
6687                let mut retry_policy = serde_json::Map::new();
6688                if let Some(max_attempts) = policy.max_attempts {
6689                    if max_attempts == 0 {
6690                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6691                            "retry_policy.max_attempts must be at least 1".to_string(),
6692                        )));
6693                    }
6694                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6695                }
6696                if !policy.backoff_seconds.is_empty() {
6697                    retry_policy
6698                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6699                }
6700                if !policy.non_retryable_error_types.is_empty() {
6701                    retry_policy.insert(
6702                        "non_retryable_error_types".to_string(),
6703                        json!(policy.non_retryable_error_types),
6704                    );
6705                }
6706                if retry_policy.is_empty() {
6707                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6708                        "retry_policy must configure at least one field".to_string(),
6709                    )));
6710                }
6711                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6712            }
6713            if let Some(seconds) = self.options.execution_timeout_seconds {
6714                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6715            }
6716            if let Some(seconds) = self.options.run_timeout_seconds {
6717                object.insert("run_timeout_seconds".to_string(), json!(seconds));
6718            }
6719            state.commands.push(command);
6720            self.scheduled = true;
6721        }
6722
6723        Poll::Pending
6724    }
6725}
6726
6727impl Future for ChildWorkflowCall {
6728    type Output = Result<ChildWorkflowResult>;
6729
6730    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6731        match self.poll_avro_value(cx) {
6732            Poll::Ready(Ok(result)) => match result.result.into_json() {
6733                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6734                    parent: result.parent,
6735                    child: result.child,
6736                    child_workflow_type: result.child_workflow_type,
6737                    result: projected,
6738                })),
6739                Err(error) => Poll::Ready(Err(error)),
6740            },
6741            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6742            Poll::Pending => Poll::Pending,
6743        }
6744    }
6745}
6746
6747fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6748    Error::NonDeterministicReplay(ReplayFailure::new(
6749        "recorded_command_mismatch",
6750        Some(recorded.sequence()),
6751        Some(recorded.shape().to_string()),
6752        Some(actual.into()),
6753        "current workflow command does not match the recorded durable command sequence",
6754    ))
6755}
6756
6757pub struct SignalCall {
6758    ctx: WorkflowContext,
6759    signal_name: String,
6760    opened_wait: bool,
6761    matched_pending: bool,
6762}
6763
6764impl SignalCall {
6765    fn poll_avro_value(
6766        mut self: Pin<&mut Self>,
6767        _cx: &mut TaskContext<'_>,
6768    ) -> Poll<Result<Vec<AvroValue>>> {
6769        if self.matched_pending {
6770            return Poll::Pending;
6771        }
6772
6773        let ctx = self.ctx.clone();
6774        let mut state = match ctx.state.lock() {
6775            Ok(state) => state,
6776            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6777        };
6778
6779        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6780            match recorded {
6781                RecordedCommand::SignalWait {
6782                    sequence,
6783                    signal_name,
6784                    value,
6785                } => {
6786                    if signal_name != self.signal_name {
6787                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6788                            ReplayFailure::new(
6789                                "recorded_command_detail_mismatch",
6790                                Some(sequence),
6791                                Some(format!("signal wait:{signal_name}")),
6792                                Some(format!("signal wait:{}", self.signal_name)),
6793                                "recorded signal name differs from the current workflow command",
6794                            ),
6795                        )));
6796                    }
6797
6798                    state.command_cursor += 1;
6799                    if let Some(value) = value {
6800                        return Poll::Ready(Ok(value));
6801                    }
6802                    if state
6803                        .resume_signal
6804                        .as_ref()
6805                        .is_some_and(|signal| signal.signal_name == self.signal_name)
6806                    {
6807                        let signal = state
6808                            .resume_signal
6809                            .take()
6810                            .expect("matching resume signal is present");
6811                        return Poll::Ready(Ok(signal.arguments));
6812                    }
6813
6814                    state.matched_recorded_pending = true;
6815                    self.opened_wait = true;
6816                    self.matched_pending = true;
6817                    return Poll::Pending;
6818                }
6819                other => {
6820                    return Poll::Ready(Err(command_mismatch(
6821                        &other,
6822                        format!("signal wait:{}", self.signal_name),
6823                    )));
6824                }
6825            }
6826        }
6827
6828        if state
6829            .resume_signal
6830            .as_ref()
6831            .is_some_and(|signal| signal.signal_name == self.signal_name)
6832        {
6833            let signal = state
6834                .resume_signal
6835                .take()
6836                .expect("matching resume signal is present");
6837            return Poll::Ready(Ok(signal.arguments));
6838        }
6839
6840        if !self.opened_wait {
6841            state.commands.push(json!({
6842                "type": "open_signal_wait",
6843                "signal_name": self.signal_name
6844            }));
6845            self.opened_wait = true;
6846        }
6847
6848        Poll::Pending
6849    }
6850}
6851
6852impl Future for SignalCall {
6853    type Output = Result<Vec<Value>>;
6854
6855    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6856        match self.poll_avro_value(cx) {
6857            Poll::Ready(Ok(values)) => Poll::Ready(
6858                values
6859                    .into_iter()
6860                    .map(AvroValue::into_json)
6861                    .collect::<Result<Vec<_>>>(),
6862            ),
6863            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6864            Poll::Pending => Poll::Pending,
6865        }
6866    }
6867}
6868
6869#[derive(Clone, Debug)]
6870pub struct ActivityContext {
6871    client: Client,
6872    pub task_id: String,
6873    pub activity_attempt_id: String,
6874    pub lease_owner: String,
6875    pub activity_type: String,
6876    pub attempt_number: u64,
6877    pub task_queue: String,
6878    pub worker_id: String,
6879}
6880
6881impl ActivityContext {
6882    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6883        self.client
6884            .heartbeat_activity_task(
6885                &self.task_id,
6886                &self.activity_attempt_id,
6887                &self.lease_owner,
6888                details,
6889            )
6890            .await
6891    }
6892}
6893
6894fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6895    match value {
6896        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6897            value, codec,
6898        )?)),
6899        None => Ok(AvroValue::Array(Vec::new())),
6900    }
6901}
6902
6903fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
6904    let Some(signal_name) = task
6905        .signal_name
6906        .as_deref()
6907        .filter(|value| !value.is_empty())
6908    else {
6909        return Ok(None);
6910    };
6911    let Some(arguments) = task.signal_arguments.as_ref() else {
6912        return Ok(None);
6913    };
6914
6915    let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
6916    let AvroValue::Array(arguments) = decoded else {
6917        unreachable!("normalize_avro_arguments always returns an array");
6918    };
6919
6920    Ok(Some(ResumeSignal {
6921        signal_name: signal_name.to_string(),
6922        arguments,
6923    }))
6924}
6925
6926fn recorded_commands(
6927    events: &[HistoryEvent],
6928    fallback_codec: &str,
6929    parent: WorkflowIdentity,
6930) -> Result<Vec<RecordedCommand>> {
6931    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
6932    let mut last_new_sequence = None;
6933
6934    for event in events {
6935        let is_activity = matches!(
6936            event.event_type.as_str(),
6937            "ActivityScheduled"
6938                | "ActivityStarted"
6939                | "ActivityHeartbeatRecorded"
6940                | "ActivityRetryScheduled"
6941                | "ActivityCompleted"
6942                | "ActivityFailed"
6943                | "ActivityCancelled"
6944                | "ActivityTimedOut"
6945        );
6946        let is_workflow_timer = matches!(
6947            event.event_type.as_str(),
6948            "TimerScheduled" | "TimerCancelled" | "TimerFired"
6949        ) && !is_internal_timer_event(event);
6950        let is_child_workflow = matches!(
6951            event.event_type.as_str(),
6952            "ChildWorkflowScheduled"
6953                | "ChildRunCompleted"
6954                | "ChildRunFailed"
6955                | "ChildRunCancelled"
6956                | "ChildRunTerminated"
6957        );
6958        let is_signal_wait = is_recorded_signal_wait_event(event);
6959        let is_side_effect = event.event_type == "SideEffectRecorded";
6960        let is_version_marker = event.event_type == "VersionMarkerRecorded";
6961        if !is_activity
6962            && !is_workflow_timer
6963            && !is_child_workflow
6964            && !is_signal_wait
6965            && !is_side_effect
6966            && !is_version_marker
6967        {
6968            continue;
6969        }
6970
6971        let sequence = durable_event_sequence(event).ok_or_else(|| {
6972            Error::NonDeterministicReplay(ReplayFailure::new(
6973                "durable_command_sequence_missing",
6974                None,
6975                Some("positive workflow sequence".to_string()),
6976                Some(event.event_type.clone()),
6977                "durable command history event has no workflow sequence",
6978            ))
6979        })?;
6980        if sequence == 0 {
6981            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6982                "durable_command_sequence_invalid",
6983                Some(sequence),
6984                Some("positive workflow sequence".to_string()),
6985                Some(sequence.to_string()),
6986                "durable command history uses an invalid workflow sequence",
6987            )));
6988        }
6989        if !events_by_sequence.contains_key(&sequence) {
6990            if let Some(previous) = last_new_sequence {
6991                if sequence < previous {
6992                    return Err(invalid_recorded_history(
6993                        "durable_command_sequence_mismatch",
6994                        sequence,
6995                        &format!("workflow sequence greater than {previous}"),
6996                        &sequence.to_string(),
6997                        "durable commands are not strictly ordered by their recorded workflow sequence",
6998                    ));
6999                }
7000            }
7001            last_new_sequence = Some(sequence);
7002        }
7003        events_by_sequence.entry(sequence).or_default().push(event);
7004    }
7005
7006    let commands: Vec<RecordedCommand> = events_by_sequence
7007        .into_iter()
7008        .map(|(sequence, sequence_events)| {
7009            let activity_events: Vec<_> = sequence_events
7010                .iter()
7011                .copied()
7012                .filter(|event| event.event_type.starts_with("Activity"))
7013                .collect();
7014            let timer_events: Vec<_> = sequence_events
7015                .iter()
7016                .copied()
7017                .filter(|event| event.event_type.starts_with("Timer"))
7018                .collect();
7019            let child_events: Vec<_> = sequence_events
7020                .iter()
7021                .copied()
7022                .filter(|event| {
7023                    event.event_type == "ChildWorkflowScheduled"
7024                        || event.event_type.starts_with("ChildRun")
7025                })
7026                .collect();
7027            let signal_wait_events: Vec<_> = sequence_events
7028                .iter()
7029                .copied()
7030                .filter(|event| is_recorded_signal_wait_event(event))
7031                .collect();
7032            let side_effect_events: Vec<_> = sequence_events
7033                .iter()
7034                .copied()
7035                .filter(|event| event.event_type == "SideEffectRecorded")
7036                .collect();
7037            let version_marker_events: Vec<_> = sequence_events
7038                .iter()
7039                .copied()
7040                .filter(|event| event.event_type == "VersionMarkerRecorded")
7041                .collect();
7042
7043            let command_kind_count = usize::from(!activity_events.is_empty())
7044                + usize::from(!timer_events.is_empty())
7045                + usize::from(!child_events.is_empty())
7046                + usize::from(!signal_wait_events.is_empty())
7047                + usize::from(!side_effect_events.is_empty())
7048                + usize::from(!version_marker_events.is_empty());
7049            if command_kind_count > 1 {
7050                let actual = [
7051                    (!activity_events.is_empty()).then_some("activity"),
7052                    (!timer_events.is_empty()).then_some("timer"),
7053                    (!child_events.is_empty()).then_some("child workflow"),
7054                    (!signal_wait_events.is_empty()).then_some("signal wait"),
7055                    (!side_effect_events.is_empty()).then_some("side effect"),
7056                    (!version_marker_events.is_empty()).then_some("version marker"),
7057                ]
7058                .into_iter()
7059                .flatten()
7060                .collect::<Vec<_>>()
7061                .join(" and ");
7062                return Err(invalid_recorded_history(
7063                    "durable_command_sequence_collision",
7064                    sequence,
7065                    "one durable command kind",
7066                    &actual,
7067                    "one workflow sequence records more than one durable command kind",
7068                ));
7069            }
7070
7071            if !activity_events.is_empty() {
7072                let scheduled_count = activity_events
7073                    .iter()
7074                    .filter(|event| event.event_type == "ActivityScheduled")
7075                    .count();
7076                if scheduled_count > 1 {
7077                    return Err(invalid_recorded_history(
7078                        "duplicate_activity_schedule",
7079                        sequence,
7080                        "at most one ActivityScheduled event",
7081                        "multiple ActivityScheduled events",
7082                        "activity history schedules more than one command at one workflow sequence",
7083                    ));
7084                }
7085                let activity_type = activity_events.iter().find_map(|event| {
7086                    event
7087                        .payload
7088                        .get("activity_type")
7089                        .or_else(|| event.payload.get("activity_name"))
7090                        .and_then(Value::as_str)
7091                        .map(str::to_string)
7092                });
7093                if activity_events.iter().filter_map(|event| {
7094                    event
7095                        .payload
7096                        .get("activity_type")
7097                        .or_else(|| event.payload.get("activity_name"))
7098                        .and_then(Value::as_str)
7099                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7100                    return Err(invalid_recorded_history(
7101                        "activity_identity_mismatch",
7102                        sequence,
7103                        activity_type.as_deref().unwrap_or("one activity identity"),
7104                        "conflicting activity identities",
7105                        "activity lifecycle events at one workflow sequence disagree on identity",
7106                    ));
7107                }
7108                let terminal: Vec<_> = activity_events
7109                    .iter()
7110                    .copied()
7111                    .filter(|event| {
7112                        matches!(
7113                            event.event_type.as_str(),
7114                            "ActivityCompleted"
7115                                | "ActivityFailed"
7116                                | "ActivityCancelled"
7117                                | "ActivityTimedOut"
7118                        )
7119                    })
7120                    .collect();
7121                if terminal.len() > 1 {
7122                    return Err(invalid_recorded_history(
7123                        "duplicate_activity_terminal_event",
7124                        sequence,
7125                        "at most one terminal activity event",
7126                        "multiple terminal activity events",
7127                        "activity history settles one command more than once",
7128                    ));
7129                }
7130                let outcome = terminal
7131                    .first()
7132                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7133                    .transpose()?;
7134                let options = activity_events
7135                    .iter()
7136                    .find(|event| event.event_type == "ActivityScheduled")
7137                    .and_then(|event| event.payload.get("activity"))
7138                    .and_then(Value::as_object)
7139                    .map(|activity| RecordedActivityOptions {
7140                        task_queue: recorded_optional_string(activity, "queue"),
7141                        execution_mode: recorded_optional_string(activity, "execution_mode"),
7142                        retry_policy: recorded_activity_retry_snapshot(
7143                            activity.get("retry_policy"),
7144                        ),
7145                    });
7146                return Ok(RecordedCommand::Activity {
7147                    sequence,
7148                    activity_type,
7149                    options,
7150                    outcome,
7151                });
7152            }
7153
7154            if !child_events.is_empty() {
7155                let scheduled: Vec<_> = child_events
7156                    .iter()
7157                    .copied()
7158                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
7159                    .collect();
7160                if scheduled.len() != 1 {
7161                    return Err(invalid_recorded_history(
7162                        "child_workflow_schedule_missing_or_duplicate",
7163                        sequence,
7164                        "one ChildWorkflowScheduled event",
7165                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7166                        "child workflow replay requires exactly one recorded schedule event",
7167                    ));
7168                }
7169                let workflow_type = child_events.iter().find_map(|event| {
7170                    event
7171                        .payload
7172                        .get("child_workflow_type")
7173                        .or_else(|| event.payload.get("workflow_type"))
7174                        .and_then(Value::as_str)
7175                        .filter(|value| !value.is_empty())
7176                        .map(str::to_string)
7177                });
7178                if child_events
7179                    .iter()
7180                    .filter_map(|event| {
7181                        event
7182                            .payload
7183                            .get("child_workflow_type")
7184                            .or_else(|| event.payload.get("workflow_type"))
7185                            .and_then(Value::as_str)
7186                    })
7187                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
7188                {
7189                    return Err(invalid_recorded_history(
7190                        "child_workflow_identity_mismatch",
7191                        sequence,
7192                        workflow_type
7193                            .as_deref()
7194                            .unwrap_or("one child workflow type"),
7195                        "conflicting child workflow types",
7196                        "child workflow lifecycle events at one sequence disagree on type",
7197                    ));
7198                }
7199                let mut outcomes = child_workflow_outcomes(
7200                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7201                    fallback_codec,
7202                    parent.clone(),
7203                )?;
7204                if outcomes.len() > 1 {
7205                    return Err(invalid_recorded_history(
7206                        "duplicate_child_workflow_terminal_event",
7207                        sequence,
7208                        "at most one terminal child event",
7209                        "multiple terminal child events",
7210                        "child workflow history settles one command more than once",
7211                    ));
7212                }
7213                return Ok(RecordedCommand::ChildWorkflow {
7214                    sequence,
7215                    workflow_type,
7216                    outcome: outcomes.pop(),
7217                });
7218            }
7219
7220            if !signal_wait_events.is_empty() {
7221                let opened: Vec<_> = signal_wait_events
7222                    .iter()
7223                    .copied()
7224                    .filter(|event| event.event_type == "SignalWaitOpened")
7225                    .collect();
7226                if opened.len() != 1 {
7227                    return Err(invalid_recorded_history(
7228                        "signal_wait_open_missing_or_duplicate",
7229                        sequence,
7230                        "one SignalWaitOpened event",
7231                        &format!("{} SignalWaitOpened events", opened.len()),
7232                        "signal replay requires exactly one canonical wait-open event",
7233                    ));
7234                }
7235
7236                let applied: Vec<_> = signal_wait_events
7237                    .iter()
7238                    .copied()
7239                    .filter(|event| event.event_type == "SignalApplied")
7240                    .collect();
7241                if applied.len() > 1 {
7242                    return Err(invalid_recorded_history(
7243                        "duplicate_signal_wait_apply",
7244                        sequence,
7245                        "at most one SignalApplied event",
7246                        "multiple SignalApplied events",
7247                        "signal history applies one durable wait more than once",
7248                    ));
7249                }
7250
7251                let signal_names = signal_wait_events
7252                    .iter()
7253                    .map(|event| required_signal_wait_name(event, sequence))
7254                    .collect::<Result<Vec<_>>>()?;
7255                let signal_name = signal_names
7256                    .first()
7257                    .expect("signal wait events are not empty")
7258                    .clone();
7259                if signal_names.iter().any(|candidate| candidate != &signal_name) {
7260                    return Err(invalid_recorded_history(
7261                        "signal_wait_identity_mismatch",
7262                        sequence,
7263                        &signal_name,
7264                        "conflicting signal names",
7265                        "signal wait lifecycle events at one workflow sequence disagree on identity",
7266                    ));
7267                }
7268                let value = applied
7269                    .first()
7270                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
7271                    .transpose()?;
7272                return Ok(RecordedCommand::SignalWait {
7273                    sequence,
7274                    signal_name,
7275                    value,
7276                });
7277            }
7278
7279            if !side_effect_events.is_empty() {
7280                if side_effect_events.len() != 1 {
7281                    return Err(invalid_recorded_history(
7282                        "duplicate_side_effect_record",
7283                        sequence,
7284                        "one SideEffectRecorded event",
7285                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
7286                        "side-effect history records one workflow command more than once",
7287                    ));
7288                }
7289                let event = side_effect_events[0];
7290                let result = event.payload.get("result").ok_or_else(|| {
7291                    invalid_recorded_history(
7292                        "side_effect_result_missing",
7293                        sequence,
7294                        "recorded result payload",
7295                        "missing result",
7296                        "side-effect history is missing its recorded value",
7297                    )
7298                })?;
7299                let has_published_envelope = result.as_str().is_some()
7300                    || result.as_object().is_some_and(|envelope| {
7301                        envelope.get("codec").and_then(Value::as_str).is_some()
7302                            && envelope.get("blob").and_then(Value::as_str).is_some()
7303                    });
7304                if !has_published_envelope {
7305                    return Err(invalid_recorded_history(
7306                        "side_effect_payload_malformed",
7307                        sequence,
7308                        "payload blob or {codec, blob} envelope",
7309                        &result.to_string(),
7310                        "side-effect history result does not use a published payload envelope",
7311                    ));
7312                }
7313                let codec = event
7314                    .payload
7315                    .get("payload_codec")
7316                    .and_then(Value::as_str)
7317                    .unwrap_or(fallback_codec);
7318                let value = decode_wire_avro_value(result, codec).map_err(|error| {
7319                    invalid_recorded_history(
7320                        "side_effect_payload_incompatible",
7321                        sequence,
7322                        &format!("valid {codec} payload envelope"),
7323                        &error.to_string(),
7324                        "side-effect history payload cannot be decoded with its recorded codec",
7325                    )
7326                })?;
7327                return Ok(RecordedCommand::SideEffect { sequence, value });
7328            }
7329
7330            if !version_marker_events.is_empty() {
7331                if version_marker_events.len() != 1 {
7332                    return Err(invalid_recorded_history(
7333                        "duplicate_version_marker_record",
7334                        sequence,
7335                        "one VersionMarkerRecorded event",
7336                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7337                        "version-marker history records one workflow command more than once",
7338                    ));
7339                }
7340                let payload = &version_marker_events[0].payload;
7341                let change_id = payload
7342                    .get("change_id")
7343                    .and_then(Value::as_str)
7344                    .filter(|value| !value.is_empty())
7345                    .map(str::to_string)
7346                    .ok_or_else(|| {
7347                        invalid_recorded_history(
7348                            "version_marker_field_missing",
7349                            sequence,
7350                            "non-empty change_id",
7351                            "missing or invalid change_id",
7352                            "version-marker history is missing its stable change ID",
7353                        )
7354                    })?;
7355                let version = required_version_i32(payload, "version", sequence)?;
7356                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7357                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7358                if min_supported > max_supported || version < min_supported || version > max_supported {
7359                    return Err(invalid_recorded_history(
7360                        "version_marker_history_range_invalid",
7361                        sequence,
7362                        "min_supported <= version <= max_supported",
7363                        &format!("{min_supported} <= {version} <= {max_supported}"),
7364                        "recorded version marker contains an internally incompatible range",
7365                    ));
7366                }
7367                return Ok(RecordedCommand::VersionMarker {
7368                    sequence,
7369                    change_id,
7370                    version,
7371                });
7372            }
7373
7374            let scheduled: Vec<_> = timer_events
7375                .iter()
7376                .copied()
7377                .filter(|event| event.event_type == "TimerScheduled")
7378                .collect();
7379            let fired: Vec<_> = timer_events
7380                .iter()
7381                .copied()
7382                .filter(|event| event.event_type == "TimerFired")
7383                .collect();
7384            if scheduled.len() != 1 {
7385                return Err(invalid_recorded_history(
7386                    "timer_schedule_missing_or_duplicate",
7387                    sequence,
7388                    "one TimerScheduled event",
7389                    &format!("{} TimerScheduled events", scheduled.len()),
7390                    "timer replay requires exactly one recorded schedule event",
7391                ));
7392            }
7393            if fired.len() > 1 {
7394                return Err(invalid_recorded_history(
7395                    "duplicate_timer_fire",
7396                    sequence,
7397                    "at most one TimerFired event",
7398                    "multiple TimerFired events",
7399                    "timer history contains more than one fire event for a workflow sequence",
7400                ));
7401            }
7402
7403            let scheduled = scheduled[0];
7404            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7405            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7406            if let Some(fired) = fired.first() {
7407                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7408                if fired_timer_id != timer_id {
7409                    return Err(invalid_recorded_history(
7410                        "timer_identity_mismatch",
7411                        sequence,
7412                        &timer_id,
7413                        &fired_timer_id,
7414                        "TimerFired does not correspond to the recorded TimerScheduled event",
7415                    ));
7416                }
7417                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7418                if fired_delay != delay_seconds {
7419                    return Err(invalid_recorded_history(
7420                        "timer_history_delay_mismatch",
7421                        sequence,
7422                        &delay_seconds.to_string(),
7423                        &fired_delay.to_string(),
7424                        "TimerScheduled and TimerFired record different delays",
7425                    ));
7426                }
7427            }
7428
7429            Ok(RecordedCommand::Timer {
7430                sequence,
7431                delay_seconds,
7432                fired: !fired.is_empty(),
7433            })
7434        })
7435        .collect::<Result<_>>()?;
7436
7437    let mut marker_sequences = HashMap::new();
7438    for command in &commands {
7439        if let RecordedCommand::VersionMarker {
7440            sequence,
7441            change_id,
7442            ..
7443        } = command
7444        {
7445            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7446                return Err(invalid_recorded_history(
7447                    "duplicate_version_marker",
7448                    *sequence,
7449                    &format!("one marker for change ID {change_id:?}"),
7450                    &format!("markers at sequences {first_sequence} and {sequence}"),
7451                    "workflow history contains duplicate markers for one stable change ID",
7452                ));
7453            }
7454        }
7455    }
7456
7457    Ok(commands)
7458}
7459
7460fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7461    payload
7462        .get(field)
7463        .and_then(Value::as_i64)
7464        .and_then(|value| i32::try_from(value).ok())
7465        .ok_or_else(|| {
7466            invalid_recorded_history(
7467                "version_marker_field_missing",
7468                sequence,
7469                &format!("integer {field}"),
7470                "missing or out-of-range integer",
7471                "version-marker history is missing a required integer field",
7472            )
7473        })
7474}
7475
7476fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7477    event
7478        .payload
7479        .get("sequence")
7480        .or_else(|| event.payload.get("workflow_sequence"))
7481        .or_else(|| event.raw.get("sequence"))
7482        .or_else(|| event.raw.get("workflow_sequence"))
7483        .and_then(value_as_u64)
7484}
7485
7486fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7487    matches!(
7488        event
7489            .payload
7490            .get("timer_kind")
7491            .or_else(|| event.raw.get("timer_kind"))
7492            .and_then(Value::as_str),
7493        Some("condition_timeout" | "signal_timeout")
7494    )
7495}
7496
7497fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7498    event
7499        .payload
7500        .get("signal_name")
7501        .or_else(|| event.raw.get("signal_name"))
7502        .and_then(Value::as_str)
7503        .filter(|value| !value.is_empty())
7504        .map(str::to_string)
7505        .ok_or_else(|| {
7506            invalid_recorded_history(
7507                "signal_wait_name_missing",
7508                sequence,
7509                "non-empty signal_name",
7510                &event.event_type,
7511                "canonical signal-wait history is missing its signal identity",
7512            )
7513        })
7514}
7515
7516fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7517    matches!(
7518        event.event_type.as_str(),
7519        "SignalWaitOpened" | "SignalApplied"
7520    )
7521}
7522
7523fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7524    event
7525        .payload
7526        .get(field)
7527        .and_then(Value::as_str)
7528        .filter(|value| !value.is_empty())
7529        .map(str::to_string)
7530        .ok_or_else(|| {
7531            invalid_recorded_history(
7532                "timer_history_field_missing",
7533                sequence,
7534                field,
7535                &event.event_type,
7536                "timer history is missing a required identity field",
7537            )
7538        })
7539}
7540
7541fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7542    event
7543        .payload
7544        .get(field)
7545        .and_then(value_as_u64)
7546        .ok_or_else(|| {
7547            invalid_recorded_history(
7548                "timer_history_field_missing",
7549                sequence,
7550                field,
7551                &event.event_type,
7552                "timer history is missing a required numeric field",
7553            )
7554        })
7555}
7556
7557fn invalid_recorded_history(
7558    reason: &str,
7559    sequence: u64,
7560    expected: &str,
7561    actual: &str,
7562    message: &str,
7563) -> Error {
7564    Error::NonDeterministicReplay(ReplayFailure::new(
7565        reason,
7566        Some(sequence),
7567        Some(expected.to_string()),
7568        Some(actual.to_string()),
7569        message,
7570    ))
7571}
7572
7573type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7574
7575fn activity_outcome(
7576    event: &HistoryEvent,
7577    fallback_codec: &str,
7578    recorded_activity_type: Option<String>,
7579) -> Result<ActivityOutcome> {
7580    if event.event_type == "ActivityCompleted" {
7581        let codec = event
7582            .payload
7583            .get("payload_codec")
7584            .and_then(Value::as_str)
7585            .unwrap_or(fallback_codec);
7586        return Ok(Ok(decode_wire_avro_value(
7587            event.payload.get("result").unwrap_or(&Value::Null),
7588            codec,
7589        )?));
7590    }
7591
7592    let payload = &event.payload;
7593    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7594        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7595        "ActivityCancelled" => (
7596            ActivityFailureKind::Cancelled,
7597            "cancelled",
7598            "activity was cancelled",
7599        ),
7600        "ActivityTimedOut" => (
7601            ActivityFailureKind::TimedOut,
7602            "timeout",
7603            "activity timed out",
7604        ),
7605        _ => unreachable!("activity_outcome is called only for terminal activity events"),
7606    };
7607    let exception = payload
7608        .get("exception")
7609        .filter(|value| !value.is_null())
7610        .cloned();
7611    let failure_category = payload_string(payload, "failure_category");
7612    let timeout_kind = payload_string(payload, "timeout_kind");
7613    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7614        ActivityFailureKind::Failed => failure_category
7615            .clone()
7616            .unwrap_or_else(|| fallback_reason.to_string()),
7617        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7618        ActivityFailureKind::TimedOut => timeout_kind
7619            .clone()
7620            .unwrap_or_else(|| fallback_reason.to_string()),
7621    });
7622    let message = payload_string(payload, "message")
7623        .or_else(|| {
7624            exception
7625                .as_ref()
7626                .and_then(|value| payload_string(value, "message"))
7627        })
7628        .unwrap_or_else(|| fallback_message.to_string());
7629
7630    Ok(Err(ActivityFailure {
7631        kind,
7632        reason,
7633        message,
7634        activity_execution_id: payload_string(payload, "activity_execution_id"),
7635        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7636        activity_type: payload_string(payload, "activity_type")
7637            .or_else(|| payload_string(payload, "activity_name"))
7638            .or(recorded_activity_type),
7639        activity_class: payload_string(payload, "activity_class"),
7640        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7641        failure_id: payload_string(payload, "failure_id"),
7642        failure_category,
7643        timeout_kind,
7644        non_retryable: payload
7645            .get("non_retryable")
7646            .and_then(Value::as_bool)
7647            .unwrap_or(false),
7648        exception_type: payload_string(payload, "exception_type").or_else(|| {
7649            exception
7650                .as_ref()
7651                .and_then(|value| payload_string(value, "type"))
7652        }),
7653        exception_class: payload_string(payload, "exception_class").or_else(|| {
7654            exception
7655                .as_ref()
7656                .and_then(|value| payload_string(value, "class"))
7657        }),
7658        code: payload
7659            .get("code")
7660            .filter(|value| !value.is_null())
7661            .cloned(),
7662        exception,
7663    }))
7664}
7665
7666type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7667
7668fn child_workflow_outcomes(
7669    events: &[HistoryEvent],
7670    fallback_codec: &str,
7671    parent: WorkflowIdentity,
7672) -> Result<Vec<ChildWorkflowOutcome>> {
7673    let mut outcomes = Vec::new();
7674
7675    for event in events {
7676        let kind = match event.event_type.as_str() {
7677            "ChildRunCompleted" => None,
7678            "ChildRunFailed" => Some((
7679                ChildWorkflowFailureKind::Failed,
7680                "child_workflow",
7681                "child workflow failed",
7682            )),
7683            "ChildRunCancelled" => Some((
7684                ChildWorkflowFailureKind::Cancelled,
7685                "cancelled",
7686                "child workflow was cancelled",
7687            )),
7688            "ChildRunTerminated" => Some((
7689                ChildWorkflowFailureKind::Terminated,
7690                "terminated",
7691                "child workflow was terminated",
7692            )),
7693            _ => continue,
7694        };
7695        let payload = &event.payload;
7696        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7697        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7698        let child_workflow_type = payload_string(payload, "child_workflow_type");
7699
7700        if let Some((kind, reason, fallback_message)) = kind {
7701            let exception = payload
7702                .get("exception")
7703                .filter(|value| !value.is_null())
7704                .cloned();
7705            let message = payload_string(payload, "message")
7706                .or_else(|| {
7707                    exception
7708                        .as_ref()
7709                        .and_then(|value| payload_string(value, "message"))
7710                })
7711                .unwrap_or_else(|| fallback_message.to_string());
7712            let exception_type = payload_string(payload, "exception_type").or_else(|| {
7713                exception
7714                    .as_ref()
7715                    .and_then(|value| payload_string(value, "type"))
7716            });
7717            let exception_class = payload_string(payload, "exception_class").or_else(|| {
7718                exception
7719                    .as_ref()
7720                    .and_then(|value| payload_string(value, "class"))
7721            });
7722            outcomes.push(Err(ChildWorkflowFailure {
7723                kind,
7724                reason: reason.to_string(),
7725                message,
7726                parent_workflow_id: parent.workflow_id.clone(),
7727                parent_workflow_run_id: parent.run_id.clone(),
7728                child_workflow_id,
7729                child_workflow_run_id,
7730                child_workflow_type,
7731                failure_id: payload_string(payload, "failure_id"),
7732                failure_category: payload_string(payload, "failure_category"),
7733                exception_type,
7734                exception_class,
7735                non_retryable: payload
7736                    .get("non_retryable")
7737                    .and_then(Value::as_bool)
7738                    .unwrap_or(false),
7739                code: payload
7740                    .get("code")
7741                    .filter(|value| !value.is_null())
7742                    .cloned(),
7743                exception,
7744            }));
7745            continue;
7746        }
7747
7748        let codec = payload
7749            .get("payload_codec")
7750            .and_then(Value::as_str)
7751            .unwrap_or(fallback_codec);
7752        let result = payload
7753            .get("result")
7754            .or_else(|| payload.get("output"))
7755            .unwrap_or(&Value::Null);
7756        outcomes.push(Ok(ChildWorkflowAvroResult {
7757            parent: parent.clone(),
7758            child: WorkflowIdentity {
7759                workflow_id: child_workflow_id,
7760                run_id: child_workflow_run_id,
7761            },
7762            child_workflow_type,
7763            result: decode_wire_avro_value(result, codec)?,
7764        }));
7765    }
7766
7767    Ok(outcomes)
7768}
7769
7770fn payload_string(payload: &Value, key: &str) -> Option<String> {
7771    payload
7772        .get(key)
7773        .and_then(Value::as_str)
7774        .filter(|value| !value.is_empty())
7775        .map(str::to_string)
7776}
7777
7778fn workflow_failure_command(error: &Error) -> Value {
7779    let (exception_type, exception_class, properties) = match error {
7780        Error::ActivityFailed(failure) => (
7781            match failure.kind {
7782                ActivityFailureKind::Failed => "ActivityFailed",
7783                ActivityFailureKind::Cancelled => "ActivityCancelled",
7784                ActivityFailureKind::TimedOut => "ActivityTimedOut",
7785            },
7786            "durable_workflow::ActivityFailure",
7787            json!({
7788                "reason": failure.reason,
7789                "activity_execution_id": failure.activity_execution_id,
7790                "activity_attempt_id": failure.activity_attempt_id,
7791                "activity_type": failure.activity_type,
7792                "activity_class": failure.activity_class,
7793                "attempt_number": failure.attempt_number,
7794                "failure_id": failure.failure_id,
7795                "failure_category": failure.failure_category,
7796                "timeout_kind": failure.timeout_kind,
7797                "activity_non_retryable": failure.non_retryable,
7798                "activity_exception_type": failure.exception_type,
7799                "activity_exception_class": failure.exception_class,
7800                "activity_code": failure.code,
7801                "activity_exception": failure.exception,
7802            }),
7803        ),
7804        Error::ChildWorkflowFailed(failure) => (
7805            match failure.kind {
7806                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7807                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7808                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7809            },
7810            "durable_workflow::ChildWorkflowFailure",
7811            json!({
7812                "reason": failure.reason,
7813                "parent_workflow_id": failure.parent_workflow_id,
7814                "parent_workflow_run_id": failure.parent_workflow_run_id,
7815                "child_workflow_id": failure.child_workflow_id,
7816                "child_workflow_run_id": failure.child_workflow_run_id,
7817                "child_workflow_type": failure.child_workflow_type,
7818                "failure_id": failure.failure_id,
7819                "failure_category": failure.failure_category,
7820                "child_exception_type": failure.exception_type,
7821                "child_exception_class": failure.exception_class,
7822                "child_non_retryable": failure.non_retryable,
7823                "child_code": failure.code,
7824                "child_exception": failure.exception,
7825            }),
7826        ),
7827        Error::NonDeterministicReplay(_) => (
7828            "NonDeterministicReplay",
7829            "durable_workflow::Error",
7830            Value::Null,
7831        ),
7832        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7833    };
7834    let non_retryable = match error {
7835        Error::ActivityFailed(failure) => failure.non_retryable,
7836        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7837        Error::NonDeterministicReplay(_) => true,
7838        _ => false,
7839    };
7840
7841    json!({
7842        "type": "fail_workflow",
7843        "message": error.to_string(),
7844        "exception_type": exception_type,
7845        "exception_class": exception_class,
7846        "non_retryable": non_retryable,
7847        "exception": {
7848            "type": exception_type,
7849            "class": exception_class,
7850            "message": error.to_string(),
7851            "properties": properties,
7852        }
7853    })
7854}
7855
7856fn workflow_task_integrity_error(error: &Error) -> bool {
7857    matches!(
7858        error,
7859        Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7860    )
7861}
7862
7863fn decode_signal_event_arguments(
7864    event: &HistoryEvent,
7865    fallback_codec: &str,
7866) -> Result<Vec<AvroValue>> {
7867    let codec = event
7868        .payload
7869        .get("payload_codec")
7870        .and_then(Value::as_str)
7871        .unwrap_or(fallback_codec);
7872    let raw = event
7873        .payload
7874        .get("value")
7875        .or_else(|| event.payload.get("input"))
7876        .or_else(|| event.payload.get("arguments"));
7877    let decoded = match raw.filter(|value| !value.is_null()) {
7878        Some(value) => decode_wire_avro_value(value, codec)?,
7879        None => AvroValue::Array(Vec::new()),
7880    };
7881    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7882        unreachable!("normalize_avro_arguments always returns an array");
7883    };
7884    Ok(arguments)
7885}
7886
7887fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7888    let Some(export_events) = task
7889        .history_export
7890        .as_ref()
7891        .and_then(|export| export.get("history_events"))
7892        .and_then(Value::as_array)
7893    else {
7894        return Ok(());
7895    };
7896
7897    if export_events.len() > task.history_events.len() {
7898        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
7899    }
7900
7901    Ok(())
7902}
7903
7904fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7905    let Some(export) = task.history_export.as_ref() else {
7906        return Ok(());
7907    };
7908    let signals = export
7909        .get("signals")
7910        .and_then(Value::as_array)
7911        .cloned()
7912        .unwrap_or_default();
7913    let activities = export
7914        .get("activities")
7915        .and_then(Value::as_array)
7916        .cloned()
7917        .unwrap_or_default();
7918    let export_codec = export
7919        .get("payloads")
7920        .and_then(|payloads| payloads.get("codec"))
7921        .and_then(Value::as_str)
7922        .unwrap_or(&task.payload_codec)
7923        .to_string();
7924    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
7925
7926    for event in &mut task.history_events {
7927        if event.event_type == "ActivityCompleted" {
7928            let sequence = event
7929                .payload
7930                .get("sequence")
7931                .or_else(|| event.payload.get("workflow_sequence"))
7932                .and_then(value_as_u64);
7933            let Some(activity) = sequence.and_then(|sequence| {
7934                activities.iter().find(|activity| {
7935                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
7936                })
7937            }) else {
7938                continue;
7939            };
7940            let Some(payload) = event.payload.as_object_mut() else {
7941                continue;
7942            };
7943            if missing_payload(payload.get("result")) {
7944                if let Some(result) = activity
7945                    .get("result")
7946                    .filter(|value| !missing_payload(Some(value)))
7947                {
7948                    payload.insert("result".to_string(), result.clone());
7949                }
7950            }
7951            for field in ["payload_codec", "activity_type"] {
7952                if payload
7953                    .get(field)
7954                    .and_then(Value::as_str)
7955                    .unwrap_or_default()
7956                    .is_empty()
7957                {
7958                    if let Some(value) = activity.get(field) {
7959                        payload.insert(field.to_string(), value.clone());
7960                    }
7961                }
7962            }
7963            continue;
7964        }
7965
7966        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
7967            continue;
7968        }
7969        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
7970        let command_id = event
7971            .payload
7972            .get("workflow_command_id")
7973            .or_else(|| event.raw.get("workflow_command_id"))
7974            .and_then(Value::as_str);
7975        let signal_name = event
7976            .payload
7977            .get("signal_name")
7978            .and_then(Value::as_str)
7979            .unwrap_or_default()
7980            .to_string();
7981        let matched = signals
7982            .iter()
7983            .find(|signal| {
7984                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
7985            })
7986            .or_else(|| {
7987                signals.iter().find(|signal| {
7988                    command_id.is_some()
7989                        && signal.get("command_id").and_then(Value::as_str) == command_id
7990                })
7991            })
7992            .or_else(|| {
7993                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
7994                let signal = signals
7995                    .iter()
7996                    .filter(|signal| {
7997                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
7998                    })
7999                    .nth(*offset);
8000                if signal.is_some() {
8001                    *offset += 1;
8002                }
8003                signal
8004            });
8005        let Some(signal) = matched else {
8006            continue;
8007        };
8008        let signal_codec = signal
8009            .get("payload_codec")
8010            .and_then(Value::as_str)
8011            .unwrap_or(&export_codec);
8012        let Some(payload) = event.payload.as_object_mut() else {
8013            continue;
8014        };
8015        if missing_payload(payload.get("arguments")) {
8016            if let Some(arguments) = signal
8017                .get("arguments")
8018                .filter(|value| !missing_payload(Some(value)))
8019            {
8020                let envelope = match arguments {
8021                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8022                    other => other.clone(),
8023                };
8024                payload.insert("arguments".to_string(), envelope);
8025            }
8026        }
8027        if payload
8028            .get("payload_codec")
8029            .and_then(Value::as_str)
8030            .unwrap_or_default()
8031            .is_empty()
8032        {
8033            payload.insert("payload_codec".to_string(), json!(signal_codec));
8034        }
8035    }
8036
8037    Ok(())
8038}
8039
8040fn missing_payload(value: Option<&Value>) -> bool {
8041    match value {
8042        None | Some(Value::Null) => true,
8043        Some(Value::String(value)) => value.is_empty(),
8044        Some(_) => false,
8045    }
8046}
8047
8048fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8049    let export_signals = task
8050        .history_export
8051        .as_ref()
8052        .and_then(|export| export.get("signals"))
8053        .and_then(Value::as_array)
8054        .cloned()
8055        .unwrap_or_default();
8056    let export_codec = task
8057        .history_export
8058        .as_ref()
8059        .and_then(|export| export.get("payloads"))
8060        .and_then(|payloads| payloads.get("codec"))
8061        .and_then(Value::as_str)
8062        .unwrap_or(&task.payload_codec);
8063    let mut name_offsets: HashMap<String, usize> = HashMap::new();
8064    let mut signals = Vec::new();
8065
8066    for event in &task.history_events {
8067        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8068            continue;
8069        }
8070
8071        let name = event
8072            .payload
8073            .get("signal_name")
8074            .and_then(Value::as_str)
8075            .unwrap_or_default();
8076        if name.is_empty() {
8077            continue;
8078        }
8079        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8080        let command_id = event
8081            .payload
8082            .get("workflow_command_id")
8083            .or_else(|| event.raw.get("workflow_command_id"))
8084            .and_then(Value::as_str);
8085        let matched_export = export_signals
8086            .iter()
8087            .find(|candidate| {
8088                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8089            })
8090            .or_else(|| {
8091                export_signals.iter().find(|candidate| {
8092                    command_id.is_some()
8093                        && candidate.get("command_id").and_then(Value::as_str) == command_id
8094                })
8095            })
8096            .or_else(|| {
8097                let offset = name_offsets.entry(name.to_string()).or_default();
8098                let candidate = export_signals
8099                    .iter()
8100                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8101                    .nth(*offset);
8102                if candidate.is_some() {
8103                    *offset += 1;
8104                }
8105                candidate
8106            });
8107        let codec = event
8108            .payload
8109            .get("payload_codec")
8110            .and_then(Value::as_str)
8111            .or_else(|| {
8112                matched_export
8113                    .and_then(|signal| signal.get("payload_codec"))
8114                    .and_then(Value::as_str)
8115            })
8116            .unwrap_or(export_codec);
8117        let raw_arguments = event
8118            .payload
8119            .get("value")
8120            .or_else(|| event.payload.get("input"))
8121            .or_else(|| event.payload.get("arguments"))
8122            .filter(|value| !value.is_null())
8123            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8124        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8125        let workflow_sequence = event
8126            .payload
8127            .get("workflow_sequence")
8128            .and_then(value_as_u64)
8129            .or_else(|| {
8130                matched_export
8131                    .and_then(|signal| signal.get("workflow_sequence"))
8132                    .and_then(value_as_u64)
8133            });
8134
8135        signals.push(QuerySignal {
8136            id: signal_id.map(str::to_string).or_else(|| {
8137                matched_export
8138                    .and_then(|signal| signal.get("id"))
8139                    .and_then(Value::as_str)
8140                    .map(str::to_string)
8141            }),
8142            name: name.to_string(),
8143            arguments,
8144            avro_arguments,
8145            workflow_sequence,
8146        });
8147    }
8148
8149    if signals.is_empty() {
8150        for signal in export_signals {
8151            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8152                continue;
8153            }
8154            let Some(name) = signal.get("name").and_then(Value::as_str) else {
8155                continue;
8156            };
8157            let codec = signal
8158                .get("payload_codec")
8159                .and_then(Value::as_str)
8160                .unwrap_or(export_codec);
8161            let (arguments, avro_arguments) =
8162                decode_query_signal_arguments(signal.get("arguments"), codec)?;
8163            signals.push(QuerySignal {
8164                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8165                name: name.to_string(),
8166                arguments,
8167                avro_arguments,
8168                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8169            });
8170        }
8171        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8172    }
8173
8174    Ok(signals)
8175}
8176
8177fn decode_query_signal_arguments(
8178    raw: Option<&Value>,
8179    codec: &str,
8180) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8181    let decoded = match raw.filter(|value| !value.is_null()) {
8182        Some(value) => decode_wire_avro_value(value, codec)?,
8183        None => AvroValue::Array(Vec::new()),
8184    };
8185    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8186        unreachable!("normalize_avro_arguments always returns an array");
8187    };
8188    let arguments = avro_arguments
8189        .iter()
8190        .cloned()
8191        .map(AvroValue::into_json)
8192        .collect::<Result<Vec<_>>>()?;
8193    Ok((arguments, avro_arguments))
8194}
8195
8196fn value_as_u64(value: &Value) -> Option<u64> {
8197    value
8198        .as_u64()
8199        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8200}
8201
8202#[cfg(test)]
8203mod tests {
8204    use super::*;
8205    use std::{
8206        io::{Read, Write},
8207        net::{SocketAddr, TcpListener, TcpStream},
8208        sync::atomic::AtomicUsize,
8209        thread,
8210    };
8211
8212    #[test]
8213    fn client_builder_rejects_the_sdk_owned_api_suffix() {
8214        for base_url in [
8215            "http://127.0.0.1:8080/api",
8216            "http://localhost:8080/api/",
8217            "https://runtime.example.test/namespaces/orders/api",
8218        ] {
8219            let error = Client::builder(base_url)
8220                .build()
8221                .expect_err("SDK-owned /api suffix must be rejected during build");
8222
8223            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8224            assert!(
8225                error.to_string().contains("SDK appends /api automatically"),
8226                "the validation error must explain how to fix the endpoint"
8227            );
8228        }
8229    }
8230
8231    #[test]
8232    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8233        for (base_url, expected) in [
8234            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8235            (
8236                "http://localhost:8080/durable-workflow/",
8237                "http://localhost:8080/durable-workflow",
8238            ),
8239            (
8240                "https://runtime.example.test/namespaces/orders",
8241                "https://runtime.example.test/namespaces/orders",
8242            ),
8243            (
8244                "https://runtime.example.test/gateway/api/namespaces/orders",
8245                "https://runtime.example.test/gateway/api/namespaces/orders",
8246            ),
8247            (
8248                "https://api.example.test/runtime/orders/",
8249                "https://api.example.test/runtime/orders",
8250            ),
8251        ] {
8252            let client = Client::builder(base_url)
8253                .build()
8254                .expect("Server and Cloud runtime base URL must remain valid");
8255
8256            assert_eq!(client.base_url, expected);
8257        }
8258    }
8259
8260    fn typed_fidelity_probe() -> AvroValue {
8261        AvroValue::Map(BTreeMap::from([
8262            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8263            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8264            (
8265                "numeric".to_string(),
8266                AvroValue::Map(BTreeMap::from([
8267                    ("0".to_string(), AvroValue::String("zero".to_string())),
8268                    ("1".to_string(), AvroValue::String("one".to_string())),
8269                ])),
8270            ),
8271            (
8272                "nested".to_string(),
8273                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8274                    "enabled".to_string(),
8275                    AvroValue::Boolean(true),
8276                )]))]),
8277            ),
8278            (
8279                "projection_collisions".to_string(),
8280                AvroValue::Array(projection_collision_probe()),
8281            ),
8282        ]))
8283    }
8284
8285    fn projection_collision_probe() -> Vec<AvroValue> {
8286        vec![
8287            AvroValue::Map(BTreeMap::from([
8288                ("$type".to_string(), AvroValue::String("bytes".to_string())),
8289                (
8290                    "base64".to_string(),
8291                    AvroValue::String("ordinary user text".to_string()),
8292                ),
8293            ])),
8294            AvroValue::Map(BTreeMap::from([
8295                ("$type".to_string(), AvroValue::String("map".to_string())),
8296                (
8297                    "entries".to_string(),
8298                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8299                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
8300                        (
8301                            "value".to_string(),
8302                            AvroValue::String("user map".to_string()),
8303                        ),
8304                    ]))]),
8305                ),
8306            ])),
8307        ]
8308    }
8309
8310    #[derive(Clone, Debug, Default, PartialEq)]
8311    struct ReplayCounterState {
8312        loaded: Option<String>,
8313        count: i64,
8314        finished: bool,
8315    }
8316
8317    fn replay_counter_worker() -> Worker {
8318        let client = Client::new("http://127.0.0.1:8080").expect("client");
8319        let mut worker = Worker::new(client, "rust-workers");
8320        worker.register_replayed_workflow(
8321            "replay-counter",
8322            ReplayCounterState::default,
8323            |ctx, _input, state| async move {
8324                let loaded = ctx.activity("load-counter", json!([])).await?;
8325                state.update(|current| {
8326                    current.loaded = loaded.as_str().map(str::to_string);
8327                })?;
8328                for _ in 0..2 {
8329                    let signal = ctx.wait_signal("increment").await?;
8330                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8331                    state.update(|current| current.count += amount)?;
8332                }
8333                state.update(|current| current.finished = true)?;
8334                state.read(|current| Ok(json!(current.count)))?
8335            },
8336        );
8337        worker.register_replayed_query::<ReplayCounterState, _, _>(
8338            "replay-counter",
8339            "current",
8340            |_ctx, state, _args| async move {
8341                Ok(json!({
8342                    "loaded": state.loaded,
8343                    "count": state.count,
8344                    "finished": state.finished,
8345                }))
8346            },
8347        );
8348        worker.register_replayed_query::<ReplayCounterState, _, _>(
8349            "replay-counter",
8350            "detached-mutation",
8351            |_ctx, state, _args| async move {
8352                let mut detached = (*state).clone();
8353                detached.count = 999;
8354                Ok(json!(detached.count))
8355            },
8356        );
8357        worker.register_replayed_query::<ReplayCounterState, _, _>(
8358            "replay-counter",
8359            "failed-mutation",
8360            |_ctx, state, _args| async move {
8361                let mut detached = (*state).clone();
8362                detached.count = 999;
8363                Err(Error::WorkerLoop("query refused".to_string()))
8364            },
8365        );
8366        worker
8367    }
8368
8369    fn replay_counter_query(
8370        query_name: &str,
8371        history_events: Value,
8372        run_status: &str,
8373    ) -> QueryTask {
8374        serde_json::from_value(json!({
8375            "query_task_id": format!("query-{query_name}"),
8376            "workflow_type": "replay-counter",
8377            "query_name": query_name,
8378            "payload_codec": "json",
8379            "workflow_arguments": {"codec": "json", "blob": "[]"},
8380            "query_arguments": {"codec": "json", "blob": "[]"},
8381            "history_events": history_events,
8382            "run_status": run_status,
8383        }))
8384        .expect("query task")
8385    }
8386
8387    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8388        workflow_context_with_codec(history, JSON_CODEC)
8389    }
8390
8391    fn workflow_context_with_codec(
8392        history: Vec<HistoryEvent>,
8393        payload_codec: &str,
8394    ) -> WorkflowContext {
8395        WorkflowContext {
8396            state: Arc::new(Mutex::new(
8397                WorkflowState::new_with_identity(
8398                    history,
8399                    None,
8400                    None,
8401                    "rust-workers".to_string(),
8402                    payload_codec.to_string(),
8403                    None,
8404                )
8405                .expect("valid workflow history"),
8406            )),
8407        }
8408    }
8409
8410    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8411        HistoryEvent {
8412            event_type: event_type.to_string(),
8413            payload,
8414            raw: HashMap::new(),
8415        }
8416    }
8417
8418    fn workflow_task(
8419        workflow_type: &str,
8420        history_events: Vec<HistoryEvent>,
8421        payload_codec: &str,
8422    ) -> WorkflowTask {
8423        WorkflowTask {
8424            task_id: format!("wft-{workflow_type}"),
8425            workflow_id: Some(format!("wf-{workflow_type}")),
8426            run_id: Some(format!("run-{workflow_type}")),
8427            workflow_type: workflow_type.to_string(),
8428            payload_codec: payload_codec.to_string(),
8429            arguments: Some(
8430                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8431            ),
8432            total_history_events: Some(history_events.len() as u64),
8433            history_size_bytes: None,
8434            continue_as_new_recommended: None,
8435            history_budget_pressure: None,
8436            history_events,
8437            next_history_page_token: None,
8438            workflow_task_attempt: 1,
8439            workflow_signal_id: None,
8440            signal_name: None,
8441            signal_arguments: None,
8442            workflow_update_id: None,
8443            update_name: None,
8444            lease_owner: Some("rust-worker".to_string()),
8445        }
8446    }
8447
8448    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8449    struct SideEffectProbe {
8450        request_id: String,
8451        attempt: u32,
8452    }
8453
8454    #[test]
8455    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8456        let calls = AtomicUsize::new(0);
8457        let ctx = workflow_context(Vec::new());
8458        let value = ctx
8459            .side_effect(|| {
8460                calls.fetch_add(1, Ordering::SeqCst);
8461                SideEffectProbe {
8462                    request_id: "request-42".to_string(),
8463                    attempt: 3,
8464                }
8465            })
8466            .expect("first side effect");
8467        assert_eq!(value.attempt, 3);
8468        assert_eq!(calls.load(Ordering::SeqCst), 1);
8469        let commands = ctx.take_commands().expect("commands");
8470        assert_eq!(commands.len(), 1);
8471        assert_eq!(commands[0]["type"], "record_side_effect");
8472        assert_eq!(
8473            decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8474            serde_json::to_value(&value).expect("value")
8475        );
8476
8477        let replay = workflow_context(vec![history_event(
8478            "SideEffectRecorded",
8479            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8480        )]);
8481        let replayed: SideEffectProbe = replay
8482            .side_effect(|| {
8483                calls.fetch_add(1, Ordering::SeqCst);
8484                panic!("committed side-effect callbacks must not run during replay")
8485            })
8486            .expect("replayed side effect");
8487        assert_eq!(replayed, value);
8488        assert_eq!(calls.load(Ordering::SeqCst), 1);
8489        assert!(replay.take_commands().expect("commands").is_empty());
8490        replay.ensure_history_consumed().expect("history consumed");
8491    }
8492
8493    #[test]
8494    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8495        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8496        let value = ctx
8497            .side_effect(|| SideEffectProbe {
8498                request_id: "avro-request".to_string(),
8499                attempt: 1,
8500            })
8501            .expect("Avro side effect");
8502        let uuid = ctx.uuid_v4().expect("deterministic UUID");
8503        let commands = ctx.take_commands().expect("commands");
8504        assert_eq!(commands.len(), 2);
8505        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8506        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8507        assert_eq!(
8508            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8509            serde_json::to_value(&value).expect("value")
8510        );
8511
8512        let replay = workflow_context_with_codec(
8513            vec![
8514                history_event(
8515                    "SideEffectRecorded",
8516                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8517                ),
8518                history_event(
8519                    "SideEffectRecorded",
8520                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8521                ),
8522            ],
8523            DEFAULT_CODEC,
8524        );
8525        let replayed: SideEffectProbe = replay
8526            .side_effect(|| panic!("Avro callback must not run"))
8527            .expect("replayed Avro value");
8528        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8529        assert_eq!(replayed, value);
8530        assert_eq!(replayed_uuid, uuid);
8531        assert!(replay.take_commands().expect("commands").is_empty());
8532    }
8533
8534    #[test]
8535    fn typed_side_effect_replay_preserves_bytes_and_maps() {
8536        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8537        let value = ctx
8538            .side_effect_avro_value(typed_fidelity_probe)
8539            .expect("typed side effect");
8540        let commands = ctx.take_commands().expect("side-effect command");
8541        assert_eq!(
8542            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8543                .expect("recorded side effect"),
8544            value
8545        );
8546
8547        let replay = workflow_context_with_codec(
8548            vec![history_event(
8549                "SideEffectRecorded",
8550                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8551            )],
8552            DEFAULT_CODEC,
8553        );
8554        assert_eq!(
8555            replay
8556                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8557                .expect("replayed typed side effect"),
8558            value
8559        );
8560    }
8561
8562    #[test]
8563    fn ordered_side_effects_share_the_durable_command_stream() {
8564        let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8565        let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8566        let ctx = workflow_context(vec![
8567            history_event(
8568                "SideEffectRecorded",
8569                json!({"sequence": 1, "result": first}),
8570            ),
8571            history_event(
8572                "SideEffectRecorded",
8573                json!({"sequence": 2, "result": second}),
8574            ),
8575        ]);
8576        let first: String = ctx
8577            .side_effect(|| panic!("first callback must not run"))
8578            .expect("first replay");
8579        let second: i32 = ctx
8580            .side_effect(|| panic!("second callback must not run"))
8581            .expect("second replay");
8582        assert_eq!(first, "first");
8583        assert_eq!(second, 29);
8584        ctx.ensure_history_consumed().expect("ordered history");
8585
8586        let reordered = workflow_context(vec![history_event(
8587            "VersionMarkerRecorded",
8588            json!({
8589                "sequence": 1,
8590                "change_id": "before-side-effect",
8591                "version": 1,
8592                "min_supported": 1,
8593                "max_supported": 1,
8594            }),
8595        )]);
8596        let error = reordered
8597            .side_effect(|| "new".to_string())
8598            .expect_err("command reordering must fail");
8599        assert!(matches!(
8600            error,
8601            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8602                if reason == "recorded_command_mismatch"
8603        ));
8604    }
8605
8606    #[test]
8607    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8608        let ctx = workflow_context(Vec::new());
8609        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8610        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8611        assert!(ctx.patched("new-search").expect("patch"));
8612        ctx.deprecate_patch("new-search").expect("deprecate patch");
8613        let commands = ctx.take_commands().expect("commands");
8614        assert_eq!(commands.len(), 2);
8615        assert_eq!(commands[0]["type"], "record_version_marker");
8616        assert_eq!(commands[0]["version"], 2);
8617        assert_eq!(commands[1]["change_id"], "new-search");
8618
8619        let replay = workflow_context(vec![history_event(
8620            "VersionMarkerRecorded",
8621            json!({
8622                "sequence": 1,
8623                "change_id": "checkout-v2",
8624                "version": 2,
8625                "min_supported": 1,
8626                "max_supported": 2,
8627            }),
8628        )]);
8629        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8630        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8631        assert!(replay.take_commands().expect("commands").is_empty());
8632        replay.ensure_history_consumed().expect("history consumed");
8633    }
8634
8635    #[test]
8636    fn version_markers_reject_incompatible_or_malformed_history() {
8637        let incompatible = workflow_context(vec![history_event(
8638            "VersionMarkerRecorded",
8639            json!({
8640                "sequence": 1,
8641                "change_id": "checkout-v2",
8642                "version": 1,
8643                "min_supported": 1,
8644                "max_supported": 2,
8645            }),
8646        )]);
8647        let error = incompatible
8648            .get_version("checkout-v2", 2, 3)
8649            .expect_err("old version is unsupported");
8650        assert!(matches!(
8651            error,
8652            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8653                if reason == "version_marker_incompatible_range"
8654        ));
8655
8656        for (history, reason) in [
8657            (
8658                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8659                "side_effect_result_missing",
8660            ),
8661            (
8662                vec![history_event(
8663                    "SideEffectRecorded",
8664                    json!({
8665                        "sequence": 1,
8666                        "result": {"codec": "avro", "blob": "not-base64"},
8667                    }),
8668                )],
8669                "side_effect_payload_incompatible",
8670            ),
8671            (
8672                vec![history_event(
8673                    "SideEffectRecorded",
8674                    json!({"sequence": 1, "result": {"unwrapped": true}}),
8675                )],
8676                "side_effect_payload_malformed",
8677            ),
8678            (
8679                vec![history_event(
8680                    "VersionMarkerRecorded",
8681                    json!({
8682                        "sequence": 1,
8683                        "change_id": "change",
8684                        "version": 1,
8685                        "min_supported": 2,
8686                        "max_supported": 1,
8687                    }),
8688                )],
8689                "version_marker_history_range_invalid",
8690            ),
8691        ] {
8692            let error = WorkflowState::new(
8693                history,
8694                "rust-workers".to_string(),
8695                JSON_CODEC.to_string(),
8696                None,
8697            )
8698            .expect_err("malformed history must fail");
8699            assert!(matches!(
8700                error,
8701                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8702                    if actual == reason
8703            ));
8704        }
8705    }
8706
8707    #[test]
8708    fn duplicate_side_effects_and_version_markers_are_rejected() {
8709        let duplicate_side_effect = WorkflowState::new(
8710            vec![
8711                history_event(
8712                    "SideEffectRecorded",
8713                    json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8714                ),
8715                history_event(
8716                    "SideEffectRecorded",
8717                    json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8718                ),
8719            ],
8720            "rust-workers".to_string(),
8721            JSON_CODEC.to_string(),
8722            None,
8723        )
8724        .expect_err("duplicate side effect");
8725        assert!(matches!(
8726            duplicate_side_effect,
8727            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8728                if reason == "duplicate_side_effect_record"
8729        ));
8730
8731        let marker = |sequence| {
8732            history_event(
8733                "VersionMarkerRecorded",
8734                json!({
8735                    "sequence": sequence,
8736                    "change_id": "same-change",
8737                    "version": 1,
8738                    "min_supported": 1,
8739                    "max_supported": 1,
8740                }),
8741            )
8742        };
8743        let duplicate_marker = WorkflowState::new(
8744            vec![marker(1), marker(3)],
8745            "rust-workers".to_string(),
8746            JSON_CODEC.to_string(),
8747            None,
8748        )
8749        .expect_err("duplicate marker");
8750        assert!(matches!(
8751            duplicate_marker,
8752            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8753                if reason == "duplicate_version_marker"
8754        ));
8755    }
8756
8757    #[test]
8758    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8759        fn worker(calls: Arc<AtomicUsize>) -> Worker {
8760            let client = Client::new("http://127.0.0.1:8080").expect("client");
8761            let mut worker = Worker::new(client, "rust-workers");
8762            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8763                let calls = Arc::clone(&calls);
8764                async move {
8765                    let captured = ctx.side_effect(|| {
8766                        calls.fetch_add(1, Ordering::SeqCst);
8767                        "captured-once".to_string()
8768                    })?;
8769                    let version = ctx.get_version("cold-restart", 1, 2)?;
8770                    Ok(json!({"captured": captured, "version": version}))
8771                }
8772            });
8773            worker
8774        }
8775
8776        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8777            WorkflowTask {
8778                task_id: "wft-side-effect-version".to_string(),
8779                workflow_id: Some("wf-side-effect-version".to_string()),
8780                run_id: Some("run-side-effect-version".to_string()),
8781                workflow_type: "rust.side-effect-version".to_string(),
8782                payload_codec: JSON_CODEC.to_string(),
8783                arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8784                history_events,
8785                total_history_events: None,
8786                history_size_bytes: None,
8787                continue_as_new_recommended: None,
8788                history_budget_pressure: None,
8789                next_history_page_token: None,
8790                workflow_task_attempt: 1,
8791                workflow_signal_id: None,
8792                signal_name: None,
8793                signal_arguments: None,
8794                workflow_update_id: None,
8795                update_name: None,
8796                lease_owner: Some("rust-worker".to_string()),
8797            }
8798        }
8799
8800        let calls = Arc::new(AtomicUsize::new(0));
8801        let initial = worker(Arc::clone(&calls))
8802            .execute_workflow_task(task(Vec::new()))
8803            .expect("initial execution");
8804        assert_eq!(
8805            initial
8806                .iter()
8807                .map(|command| &command["type"])
8808                .collect::<Vec<_>>(),
8809            vec![
8810                "record_side_effect",
8811                "record_version_marker",
8812                "complete_workflow"
8813            ]
8814        );
8815        assert_eq!(calls.load(Ordering::SeqCst), 1);
8816
8817        let restarted = worker(Arc::clone(&calls));
8818        let replayed = restarted
8819            .execute_workflow_task(task(vec![
8820                history_event(
8821                    "SideEffectRecorded",
8822                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8823                ),
8824                history_event(
8825                    "VersionMarkerRecorded",
8826                    json!({
8827                        "sequence": 2,
8828                        "change_id": "cold-restart",
8829                        "version": 2,
8830                        "min_supported": 1,
8831                        "max_supported": 2,
8832                    }),
8833                ),
8834            ]))
8835            .expect("cold replay");
8836        assert_eq!(replayed.len(), 1);
8837        assert_eq!(replayed[0]["type"], "complete_workflow");
8838        assert_eq!(calls.load(Ordering::SeqCst), 1);
8839    }
8840
8841    #[test]
8842    fn side_effect_replay_rejects_changed_rust_value_type() {
8843        let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8844        let ctx = workflow_context(vec![history_event(
8845            "SideEffectRecorded",
8846            json!({"sequence": 1, "result": result}),
8847        )]);
8848        let error = ctx
8849            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8850            .expect_err("changed type must fail replay");
8851        assert!(matches!(
8852            error,
8853            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8854                if reason == "side_effect_type_mismatch"
8855        ));
8856    }
8857
8858    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8859        vec![
8860            history_event(
8861                "ActivityScheduled",
8862                json!({
8863                    "sequence": 1,
8864                    "activity_type": "flaky",
8865                    "activity_execution_id": "act-1",
8866                    "activity": {
8867                        "id": "act-1",
8868                        "sequence": 1,
8869                        "type": "flaky",
8870                        "queue": "critical-activities",
8871                        "execution_mode": null,
8872                        "retry_policy": {
8873                            "snapshot_version": 1,
8874                            "max_attempts": 3,
8875                            "backoff_seconds": [2, 4],
8876                            "start_to_close_timeout": 30,
8877                            "schedule_to_start_timeout": 5,
8878                            "schedule_to_close_timeout": 90,
8879                            "heartbeat_timeout": 10,
8880                            "non_retryable_error_types": ["PermanentError"]
8881                        }
8882                    }
8883                }),
8884            ),
8885            history_event(
8886                "ActivityStarted",
8887                json!({
8888                    "sequence": 1,
8889                    "activity_type": "flaky",
8890                    "activity_execution_id": "act-1",
8891                    "activity_attempt_id": "attempt-1",
8892                    "attempt_number": 1
8893                }),
8894            ),
8895            history_event(
8896                "ActivityRetryScheduled",
8897                json!({
8898                    "sequence": 1,
8899                    "activity_type": "flaky",
8900                    "activity_execution_id": "act-1",
8901                    "activity_attempt_id": "attempt-1",
8902                    "attempt_number": 1,
8903                    "retry_after_attempt": 1,
8904                    "retry_backoff_seconds": 2,
8905                    "failure_category": "activity",
8906                    "exception_type": "TransientError"
8907                }),
8908            ),
8909            history_event(
8910                "ActivityStarted",
8911                json!({
8912                    "sequence": 1,
8913                    "activity_type": "flaky",
8914                    "activity_execution_id": "act-1",
8915                    "activity_attempt_id": "attempt-2",
8916                    "attempt_number": 2
8917                }),
8918            ),
8919            history_event(
8920                "ActivityCompleted",
8921                json!({
8922                    "sequence": 1,
8923                    "activity_type": "flaky",
8924                    "activity_execution_id": "act-1",
8925                    "activity_attempt_id": "attempt-2",
8926                    "attempt_number": 2,
8927                    "payload_codec": "json",
8928                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
8929                }),
8930            ),
8931        ]
8932    }
8933
8934    fn retry_activity_options() -> ActivityOptions {
8935        ActivityOptions::new()
8936            .task_queue("critical-activities")
8937            .retry_policy(
8938                ActivityRetryPolicy::new(3)
8939                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
8940                    .non_retryable_error_type("PermanentError"),
8941            )
8942            .start_to_close_timeout(Duration::from_secs(30))
8943            .schedule_to_start_timeout(Duration::from_secs(5))
8944            .schedule_to_close_timeout(Duration::from_secs(90))
8945            .heartbeat_timeout(Duration::from_secs(10))
8946    }
8947
8948    #[test]
8949    fn fixed_avro_value_round_trips_json_values() {
8950        let value = json!({"greeting": "hello", "count": 3, "ok": true});
8951        let envelope = PayloadEnvelope::avro(&value).expect("encode");
8952        assert_eq!(envelope.codec, DEFAULT_CODEC);
8953        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
8954    }
8955
8956    #[tokio::test]
8957    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
8958        let client = Client::new("http://127.0.0.1:8080").expect("client");
8959        let mut worker = Worker::new(client, "rust-workers");
8960        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
8961        worker
8962            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
8963        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
8964            Ok(input)
8965        });
8966        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
8967            Ok(input)
8968        });
8969        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
8970            Ok(AvroValue::Array(
8971                ctx.wait_signal_avro_value("changed").await?,
8972            ))
8973        });
8974
8975        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
8976        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
8977
8978        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
8979        workflow.arguments = Some(envelope.clone());
8980        let commands = worker
8981            .execute_workflow_task(workflow)
8982            .expect("typed workflow task");
8983        assert_eq!(commands[0]["type"], "complete_workflow");
8984        assert_eq!(
8985            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8986                .expect("typed workflow result"),
8987            arguments
8988        );
8989
8990        let activity = ActivityTask {
8991            task_id: "activity-typed".to_string(),
8992            activity_attempt_id: Some("attempt-typed".to_string()),
8993            attempt_id: None,
8994            activity_type: "typed.activity".to_string(),
8995            payload_codec: DEFAULT_CODEC.to_string(),
8996            arguments: Some(envelope.clone()),
8997            attempt_number: 1,
8998            lease_owner: Some("rust-worker".to_string()),
8999        };
9000        assert_eq!(
9001            worker
9002                .execute_activity_task(activity)
9003                .await
9004                .expect("typed activity result"),
9005            arguments
9006        );
9007
9008        let query = QueryTask {
9009            query_task_id: "query-typed".to_string(),
9010            query_task_attempt: 1,
9011            lease_owner: Some("rust-worker".to_string()),
9012            workflow_id: Some("typed-1".to_string()),
9013            run_id: Some("run-typed".to_string()),
9014            workflow_type: "typed.echo".to_string(),
9015            query_name: "inspect".to_string(),
9016            payload_codec: DEFAULT_CODEC.to_string(),
9017            workflow_arguments: Some(
9018                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9019                    .expect("workflow input"),
9020            ),
9021            query_arguments: Some(envelope.clone()),
9022            history_events: Vec::new(),
9023            history_export: None,
9024            run_status: Some("running".to_string()),
9025        };
9026        assert_eq!(
9027            worker
9028                .execute_query_task(query)
9029                .await
9030                .expect("typed query result"),
9031            arguments
9032        );
9033
9034        let mut update = workflow_task(
9035            "typed.echo",
9036            vec![history_event(
9037                "UpdateAccepted",
9038                json!({
9039                    "update_id": "update-typed",
9040                    "update_name": "replace",
9041                    "arguments": envelope.clone(),
9042                }),
9043            )],
9044            DEFAULT_CODEC,
9045        );
9046        update.workflow_update_id = Some("update-typed".to_string());
9047        update.update_name = Some("replace".to_string());
9048        let commands = worker
9049            .execute_workflow_task(update)
9050            .expect("typed update task");
9051        assert_eq!(commands[0]["type"], "complete_update");
9052        assert_eq!(
9053            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9054                .expect("typed update result"),
9055            arguments
9056        );
9057
9058        let mut signal = workflow_task(
9059            "typed.signal",
9060            vec![history_event(
9061                "SignalReceived",
9062                json!({
9063                    "signal_id": "signal-typed",
9064                    "signal_name": "changed",
9065                    "arguments": envelope.clone(),
9066                }),
9067            )],
9068            DEFAULT_CODEC,
9069        );
9070        signal.workflow_signal_id = Some("signal-typed".to_string());
9071        signal.signal_name = Some("changed".to_string());
9072        signal.signal_arguments = Some(envelope);
9073        let commands = worker
9074            .execute_workflow_task(signal)
9075            .expect("typed signal resume");
9076        assert_eq!(
9077            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9078                .expect("typed signal result"),
9079            arguments
9080        );
9081    }
9082
9083    #[tokio::test]
9084    async fn typed_helpers_never_parse_json_inspection_projection() {
9085        let collision_values = projection_collision_probe();
9086        let expected = AvroValue::Array(collision_values.clone());
9087        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9088
9089        let activity_context = workflow_context_with_codec(
9090            vec![history_event(
9091                "ActivityCompleted",
9092                json!({
9093                    "sequence": 1,
9094                    "activity_type": "collision.activity",
9095                    "payload_codec": DEFAULT_CODEC,
9096                    "result": envelope.clone(),
9097                }),
9098            )],
9099            DEFAULT_CODEC,
9100        );
9101        assert_eq!(
9102            activity_context
9103                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9104                .await
9105                .expect("typed activity collision result"),
9106            expected
9107        );
9108
9109        let signal_context = workflow_context_with_codec(
9110            vec![
9111                history_event(
9112                    "SignalWaitOpened",
9113                    json!({"sequence": 1, "signal_name": "collision"}),
9114                ),
9115                history_event(
9116                    "SignalApplied",
9117                    json!({
9118                        "sequence": 1,
9119                        "signal_name": "collision",
9120                        "payload_codec": DEFAULT_CODEC,
9121                        "value": envelope.clone(),
9122                    }),
9123                ),
9124            ],
9125            DEFAULT_CODEC,
9126        );
9127        assert_eq!(
9128            signal_context
9129                .wait_signal_avro_value("collision")
9130                .await
9131                .expect("typed signal collision arguments"),
9132            collision_values
9133        );
9134
9135        let child_context = workflow_context_with_codec(
9136            vec![
9137                history_event(
9138                    "ChildWorkflowScheduled",
9139                    json!({
9140                        "sequence": 1,
9141                        "child_workflow_instance_id": "collision-child",
9142                        "child_workflow_run_id": "collision-run",
9143                        "child_workflow_type": "collision.child",
9144                    }),
9145                ),
9146                history_event(
9147                    "ChildRunCompleted",
9148                    json!({
9149                        "sequence": 1,
9150                        "child_workflow_instance_id": "collision-child",
9151                        "child_workflow_run_id": "collision-run",
9152                        "child_workflow_type": "collision.child",
9153                        "payload_codec": DEFAULT_CODEC,
9154                        "result": envelope,
9155                    }),
9156                ),
9157            ],
9158            DEFAULT_CODEC,
9159        );
9160        let child = child_context
9161            .start_child_workflow_avro_value(
9162                "collision.child",
9163                ChildWorkflowOptions::new("collision-workers"),
9164                AvroValue::Array(Vec::new()),
9165            )
9166            .await
9167            .expect("typed child collision result");
9168        assert_eq!(child.result, expected);
9169    }
9170
9171    #[tokio::test]
9172    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9173        let client = Client::new("http://127.0.0.1:8080").expect("client");
9174        let mut worker = Worker::new(client, "rust-workers");
9175        worker.register_replayed_workflow_avro_value(
9176            "typed.replayed",
9177            || (),
9178            |_ctx, input, _state| async move { Ok(input) },
9179        );
9180        worker.register_replayed_query_avro_value::<(), _, _>(
9181            "typed.replayed",
9182            "inspect",
9183            |ctx, _state, args| async move {
9184                let mut signals = ctx.signals_avro_value("collision");
9185                let signal = signals
9186                    .pop()
9187                    .map(AvroValue::Array)
9188                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9189                Ok(AvroValue::Array(vec![
9190                    ctx.workflow_input_avro_value().clone(),
9191                    signal,
9192                    args,
9193                ]))
9194            },
9195        );
9196        let arguments = AvroValue::Array(projection_collision_probe());
9197        let signal_arguments =
9198            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9199        let task = QueryTask {
9200            query_task_id: "query-typed-replay".to_string(),
9201            query_task_attempt: 1,
9202            lease_owner: Some("rust-worker".to_string()),
9203            workflow_id: Some("typed-replay".to_string()),
9204            run_id: Some("run-typed-replay".to_string()),
9205            workflow_type: "typed.replayed".to_string(),
9206            query_name: "inspect".to_string(),
9207            payload_codec: DEFAULT_CODEC.to_string(),
9208            workflow_arguments: Some(
9209                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9210            ),
9211            query_arguments: Some(
9212                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9213            ),
9214            history_events: vec![history_event(
9215                "SignalReceived",
9216                json!({
9217                    "signal_id": "collision-signal",
9218                    "signal_name": "collision",
9219                    "workflow_sequence": 1,
9220                    "payload_codec": DEFAULT_CODEC,
9221                    "arguments": signal_arguments,
9222                }),
9223            )],
9224            history_export: None,
9225            run_status: Some("completed".to_string()),
9226        };
9227
9228        assert_eq!(
9229            worker
9230                .execute_query_task(task)
9231                .await
9232                .expect("typed replay query"),
9233            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9234        );
9235    }
9236
9237    #[test]
9238    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9239        let value = BTreeMap::from([(1_i32, "integer key")]);
9240        let error = PayloadEnvelope::avro(&value)
9241            .expect_err("integer map keys must fail")
9242            .to_string();
9243
9244        assert!(error.contains("invalid_map_key"));
9245    }
9246
9247    #[test]
9248    fn json_codec_remains_plain_json() {
9249        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9250        let envelope = PayloadEnvelope::json(&value).expect("encode");
9251
9252        assert_eq!(envelope.codec, JSON_CODEC);
9253        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9254        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9255    }
9256
9257    #[test]
9258    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9259        let envelope = PayloadEnvelope {
9260            codec: DEFAULT_CODEC.to_string(),
9261            blob: BASE64.encode([0x01]),
9262        };
9263
9264        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9265        assert!(error.to_string().contains("invalid_payload_framing"));
9266    }
9267
9268    #[test]
9269    fn workflow_context_schedules_activity_until_completion_is_in_history() {
9270        let ctx = WorkflowContext {
9271            state: Arc::new(Mutex::new(
9272                WorkflowState::new_with_identity(
9273                    Vec::new(),
9274                    Some("wf-parent".to_string()),
9275                    Some("run-parent".to_string()),
9276                    "rust-workers".to_string(),
9277                    DEFAULT_CODEC.to_string(),
9278                    None,
9279                )
9280                .expect("workflow state"),
9281            )),
9282        };
9283
9284        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9285        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9286        assert!(matches!(
9287            call.as_mut().poll(&mut task_context),
9288            Poll::Pending
9289        ));
9290
9291        let commands = ctx.take_commands().expect("commands");
9292        assert_eq!(commands[0]["type"], "schedule_activity");
9293        assert_eq!(commands[0]["activity_type"], "hello.activity");
9294    }
9295
9296    #[test]
9297    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9298        let ctx = workflow_context(Vec::new());
9299        let options = ActivityOptions::new()
9300            .task_queue("payments")
9301            .retry_policy(
9302                ActivityRetryPolicy::new(4)
9303                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9304                    .non_retryable_error_type("ValidationError"),
9305            )
9306            .start_to_close_timeout(Duration::from_secs(120))
9307            .schedule_to_start_timeout(Duration::from_secs(10))
9308            .schedule_to_close_timeout(Duration::from_secs(300))
9309            .heartbeat_timeout(Duration::from_secs(15));
9310        let mut call = Box::pin(ctx.activity_with_options(
9311            "charge-card",
9312            options,
9313            json!([{"order_id": "o-1"}]),
9314        ));
9315        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9316
9317        assert!(matches!(
9318            call.as_mut().poll(&mut task_context),
9319            Poll::Pending
9320        ));
9321        assert!(matches!(
9322            call.as_mut().poll(&mut task_context),
9323            Poll::Pending
9324        ));
9325
9326        let commands = ctx.take_commands().expect("activity command");
9327        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9328        assert_eq!(commands[0]["queue"], "payments");
9329        assert_eq!(
9330            commands[0]["retry_policy"],
9331            json!({
9332                "max_attempts": 4,
9333                "backoff_seconds": [1, 3, 9],
9334                "non_retryable_error_types": ["ValidationError"],
9335            })
9336        );
9337        assert_eq!(commands[0]["start_to_close_timeout"], 120);
9338        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9339        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9340        assert_eq!(commands[0]["heartbeat_timeout"], 15);
9341    }
9342
9343    #[test]
9344    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9345        let ctx = workflow_context(Vec::new());
9346        let options = ActivityOptions::new().retry_policy(
9347            ActivityRetryPolicy::new(3)
9348                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9349        );
9350        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9351        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9352
9353        assert!(matches!(
9354            call.as_mut().poll(&mut task_context),
9355            Poll::Pending
9356        ));
9357        assert_eq!(
9358            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9359            json!([1, 2])
9360        );
9361    }
9362
9363    #[test]
9364    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9365        let cases = [
9366            (
9367                ActivityOptions::new().task_queue("  "),
9368                ActivityOptionsErrorKind::EmptyTaskQueue,
9369            ),
9370            (
9371                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9372                ActivityOptionsErrorKind::EmptyRetryPolicy,
9373            ),
9374            (
9375                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9376                ActivityOptionsErrorKind::InvalidMaxAttempts,
9377            ),
9378            (
9379                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9380                    max_attempts: None,
9381                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9382                    non_retryable_error_types: Vec::new(),
9383                }),
9384                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9385            ),
9386            (
9387                ActivityOptions::new().retry_policy(
9388                    ActivityRetryPolicy::new(2)
9389                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9390                ),
9391                ActivityOptionsErrorKind::TooManyBackoffIntervals,
9392            ),
9393            (
9394                ActivityOptions::new().retry_policy(
9395                    ActivityRetryPolicy::new(2).exponential_backoff(
9396                        Duration::from_secs(1),
9397                        0,
9398                        None,
9399                    ),
9400                ),
9401                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9402            ),
9403            (
9404                ActivityOptions::new()
9405                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
9406                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9407            ),
9408            (
9409                ActivityOptions::new().retry_policy(
9410                    ActivityRetryPolicy::new(10_002).exponential_backoff(
9411                        Duration::from_secs(1),
9412                        1,
9413                        None,
9414                    ),
9415                ),
9416                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9417            ),
9418            (
9419                ActivityOptions::new().retry_policy(
9420                    ActivityRetryPolicy::new(2)
9421                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9422                ),
9423                ActivityOptionsErrorKind::BackoffOverflow,
9424            ),
9425        ];
9426
9427        for (options, expected_kind) in cases {
9428            let ctx = workflow_context(Vec::new());
9429            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9430            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9431            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9432                call.as_mut().poll(&mut task_context)
9433            else {
9434                panic!("expected typed activity validation error");
9435            };
9436            assert_eq!(error.kind, expected_kind);
9437            assert!(ctx.take_commands().expect("commands").is_empty());
9438        }
9439    }
9440
9441    #[test]
9442    fn activity_options_validate_positive_and_ordered_timeouts() {
9443        let zero_timeout_cases = [
9444            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9445            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9446            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9447            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9448        ];
9449        for options in zero_timeout_cases {
9450            assert_eq!(
9451                options.validate().expect_err("zero timeout").kind,
9452                ActivityOptionsErrorKind::TimeoutNotPositive
9453            );
9454        }
9455
9456        let ordering_cases = [
9457            ActivityOptions::new()
9458                .heartbeat_timeout(Duration::from_secs(11))
9459                .start_to_close_timeout(Duration::from_secs(10)),
9460            ActivityOptions::new()
9461                .start_to_close_timeout(Duration::from_secs(31))
9462                .schedule_to_close_timeout(Duration::from_secs(30)),
9463            ActivityOptions::new()
9464                .schedule_to_start_timeout(Duration::from_secs(31))
9465                .schedule_to_close_timeout(Duration::from_secs(30)),
9466        ];
9467        for options in ordering_cases {
9468            assert_eq!(
9469                options.validate().expect_err("timeout order").kind,
9470                ActivityOptionsErrorKind::TimeoutOrder
9471            );
9472        }
9473
9474        assert_eq!(
9475            ActivityOptions::new()
9476                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9477                .validate()
9478                .expect_err("protocol integer overflow")
9479                .kind,
9480            ActivityOptionsErrorKind::TimeoutOverflow
9481        );
9482    }
9483
9484    #[test]
9485    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9486        let ctx = workflow_context(completed_retry_activity_history());
9487        let mut call =
9488            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9489        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9490
9491        assert!(matches!(
9492            call.as_mut().poll(&mut task_context),
9493            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9494        ));
9495        assert!(ctx.take_commands().expect("commands").is_empty());
9496        ctx.ensure_history_consumed().expect("history consumed");
9497    }
9498
9499    #[test]
9500    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9501        let mut options = retry_activity_options();
9502        options
9503            .retry_policy
9504            .as_mut()
9505            .expect("retry policy")
9506            .non_retryable_error_types
9507            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9508
9509        let new_ctx = workflow_context(Vec::new());
9510        let mut new_call =
9511            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9512        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9513        assert!(matches!(
9514            new_call.as_mut().poll(&mut task_context),
9515            Poll::Pending
9516        ));
9517        let commands = new_ctx.take_commands().expect("commands");
9518        assert_eq!(commands.len(), 1);
9519        assert_eq!(
9520            commands[0]["retry_policy"]["non_retryable_error_types"],
9521            json!(["PermanentError"])
9522        );
9523
9524        let replay_ctx = workflow_context(completed_retry_activity_history());
9525        let mut replay_call =
9526            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9527        assert!(matches!(
9528            replay_call.as_mut().poll(&mut task_context),
9529            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9530        ));
9531        assert!(replay_ctx.take_commands().expect("commands").is_empty());
9532        replay_ctx
9533            .ensure_history_consumed()
9534            .expect("history consumed");
9535    }
9536
9537    #[test]
9538    fn replayed_intermediate_retry_remains_pending_across_restarts() {
9539        let history = completed_retry_activity_history()
9540            .into_iter()
9541            .take(3)
9542            .collect::<Vec<_>>();
9543
9544        for _restart in 0..2 {
9545            let ctx = workflow_context(history.clone());
9546            let mut call =
9547                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9548            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9549            assert!(matches!(
9550                call.as_mut().poll(&mut task_context),
9551                Poll::Pending
9552            ));
9553            assert!(ctx.take_commands().expect("commands").is_empty());
9554        }
9555    }
9556
9557    #[test]
9558    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9559        let mut changed_queue = retry_activity_options();
9560        changed_queue.task_queue = Some("different-queue".to_string());
9561
9562        let mut changed_max_attempts = retry_activity_options();
9563        let retry_policy = changed_max_attempts
9564            .retry_policy
9565            .as_mut()
9566            .expect("retry policy");
9567        retry_policy.max_attempts = Some(4);
9568
9569        let mut changed_backoff = retry_activity_options();
9570        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9571        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9572            Duration::from_secs(3),
9573            Duration::from_secs(4),
9574        ]));
9575
9576        let mut changed_non_retryable_types = retry_activity_options();
9577        let retry_policy = changed_non_retryable_types
9578            .retry_policy
9579            .as_mut()
9580            .expect("retry policy");
9581        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9582
9583        let mut changed_start_to_close = retry_activity_options();
9584        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9585        let mut changed_schedule_to_start = retry_activity_options();
9586        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9587        let mut changed_schedule_to_close = retry_activity_options();
9588        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9589        let mut changed_heartbeat = retry_activity_options();
9590        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9591
9592        let cases = [
9593            (changed_queue, "activity_task_queue_mismatch"),
9594            (changed_max_attempts, "activity_retry_policy_mismatch"),
9595            (changed_backoff, "activity_retry_policy_mismatch"),
9596            (
9597                changed_non_retryable_types,
9598                "activity_retry_policy_mismatch",
9599            ),
9600            (changed_start_to_close, "activity_retry_policy_mismatch"),
9601            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9602            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9603            (changed_heartbeat, "activity_retry_policy_mismatch"),
9604        ];
9605
9606        for (options, expected_reason) in cases {
9607            let ctx = workflow_context(completed_retry_activity_history());
9608            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9609            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9610            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9611                call.as_mut().poll(&mut task_context)
9612            else {
9613                panic!("changed activity options must fail replay");
9614            };
9615            assert_eq!(failure.reason, expected_reason);
9616            assert_eq!(failure.sequence, Some(1));
9617            assert!(ctx.take_commands().expect("commands").is_empty());
9618        }
9619    }
9620
9621    #[test]
9622    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9623        let cases = [
9624            (
9625                "execution_mode",
9626                json!("local"),
9627                "activity_execution_mode_mismatch",
9628            ),
9629            (
9630                "snapshot_version",
9631                json!(2),
9632                "activity_retry_policy_mismatch",
9633            ),
9634        ];
9635
9636        for (field, value, expected_reason) in cases {
9637            let mut history = completed_retry_activity_history();
9638            let activity = history[0].payload["activity"]
9639                .as_object_mut()
9640                .expect("activity snapshot");
9641            if field == "execution_mode" {
9642                activity.insert(field.to_string(), value);
9643            } else {
9644                activity["retry_policy"]
9645                    .as_object_mut()
9646                    .expect("retry snapshot")
9647                    .insert(field.to_string(), value);
9648            }
9649
9650            let ctx = workflow_context(history);
9651            let mut call =
9652                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9653            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9654            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9655                call.as_mut().poll(&mut task_context)
9656            else {
9657                panic!("changed {field} must fail replay");
9658            };
9659            assert_eq!(failure.reason, expected_reason);
9660            assert_eq!(failure.sequence, Some(1));
9661            assert!(ctx.take_commands().expect("commands").is_empty());
9662        }
9663    }
9664
9665    #[test]
9666    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9667        let mut history = completed_retry_activity_history();
9668        let activity = history[0].payload["activity"]
9669            .as_object_mut()
9670            .expect("activity snapshot");
9671        activity.remove("execution_mode");
9672        activity.remove("retry_policy");
9673
9674        let mut current = retry_activity_options();
9675        current.start_to_close_timeout = Some(Duration::from_secs(45));
9676        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9677        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9678        current.heartbeat_timeout = Some(Duration::from_secs(12));
9679
9680        let ctx = workflow_context(history);
9681        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9682        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9683        assert!(matches!(
9684            call.as_mut().poll(&mut task_context),
9685            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9686        ));
9687        assert!(ctx.take_commands().expect("commands").is_empty());
9688        ctx.ensure_history_consumed().expect("history consumed");
9689    }
9690
9691    #[test]
9692    fn terminal_activity_failed_after_start_returns_typed_failure() {
9693        let history = vec![
9694            history_event(
9695                "ActivityScheduled",
9696                json!({
9697                    "sequence": 1,
9698                    "activity_type": "flaky",
9699                    "activity_execution_id": "act-terminal",
9700                    "activity": {
9701                        "id": "act-terminal",
9702                        "sequence": 1,
9703                        "type": "flaky",
9704                        "queue": "critical-activities",
9705                        "retry_policy": {
9706                            "snapshot_version": 1,
9707                            "max_attempts": 3,
9708                            "backoff_seconds": [2, 4],
9709                            "non_retryable_error_types": ["PermanentError"]
9710                        }
9711                    }
9712                }),
9713            ),
9714            history_event(
9715                "ActivityStarted",
9716                json!({
9717                    "sequence": 1,
9718                    "activity_type": "flaky",
9719                    "activity_execution_id": "act-terminal",
9720                    "activity_attempt_id": "attempt-1",
9721                    "attempt_number": 1
9722                }),
9723            ),
9724            history_event(
9725                "ActivityFailed",
9726                json!({
9727                    "sequence": 1,
9728                    "activity_type": "flaky",
9729                    "activity_execution_id": "act-terminal",
9730                    "activity_attempt_id": "attempt-1",
9731                    "attempt_number": 1,
9732                    "failure_id": "failure-terminal",
9733                    "failure_category": "activity",
9734                    "exception_type": "PermanentError",
9735                    "message": "cannot retry",
9736                    "non_retryable": true
9737                }),
9738            ),
9739        ];
9740        let ctx = workflow_context(history);
9741        let mut call =
9742            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9743        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9744
9745        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9746            call.as_mut().poll(&mut task_context)
9747        else {
9748            panic!("terminal ActivityFailed must settle the activity future");
9749        };
9750        assert_eq!(failure.kind, ActivityFailureKind::Failed);
9751        assert_eq!(
9752            failure.activity_execution_id.as_deref(),
9753            Some("act-terminal")
9754        );
9755        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9756        assert!(failure.non_retryable);
9757        assert!(ctx.take_commands().expect("commands").is_empty());
9758        ctx.ensure_history_consumed().expect("history consumed");
9759    }
9760
9761    #[test]
9762    fn activity_terminal_events_return_machine_readable_failures() {
9763        let cases = [
9764            (
9765                "ActivityFailed",
9766                json!({
9767                    "sequence": 1,
9768                    "activity_type": "charge-card",
9769                    "activity_execution_id": "act-1",
9770                    "activity_attempt_id": "attempt-2",
9771                    "attempt_number": 2,
9772                    "failure_id": "failure-1",
9773                    "failure_category": "activity",
9774                    "exception_type": "PaymentDeclined",
9775                    "exception_class": "payments.PaymentDeclined",
9776                    "message": "card declined",
9777                    "non_retryable": true
9778                }),
9779                ActivityFailureKind::Failed,
9780                "activity",
9781            ),
9782            (
9783                "ActivityCancelled",
9784                json!({
9785                    "sequence": 1,
9786                    "activity_type": "charge-card",
9787                    "activity_execution_id": "act-1",
9788                    "activity_attempt_id": "attempt-1"
9789                }),
9790                ActivityFailureKind::Cancelled,
9791                "cancelled",
9792            ),
9793        ];
9794
9795        for (event_type, payload, expected_kind, expected_reason) in cases {
9796            let ctx = workflow_context(vec![history_event(event_type, payload)]);
9797            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9798            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9799            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9800                call.as_mut().poll(&mut task_context)
9801            else {
9802                panic!("expected terminal activity failure");
9803            };
9804            assert_eq!(failure.kind, expected_kind);
9805            assert_eq!(failure.reason, expected_reason);
9806            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9807            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9808        }
9809    }
9810
9811    #[test]
9812    fn every_activity_timeout_class_is_typed() {
9813        for timeout_kind in [
9814            "start_to_close",
9815            "schedule_to_start",
9816            "schedule_to_close",
9817            "heartbeat",
9818        ] {
9819            let ctx = workflow_context(vec![history_event(
9820                "ActivityTimedOut",
9821                json!({
9822                    "sequence": 1,
9823                    "activity_type": "slow",
9824                    "activity_execution_id": "act-timeout",
9825                    "activity_attempt_id": "attempt-timeout",
9826                    "failure_category": "timeout",
9827                    "timeout_kind": timeout_kind,
9828                    "message": "deadline expired"
9829                }),
9830            )]);
9831            let mut call = Box::pin(ctx.activity("slow", json!([])));
9832            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9833            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9834                call.as_mut().poll(&mut task_context)
9835            else {
9836                panic!("expected timeout failure");
9837            };
9838            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9839            assert_eq!(failure.reason, timeout_kind);
9840            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9841            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9842        }
9843    }
9844
9845    #[test]
9846    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9847        let ctx = workflow_context(Vec::new());
9848        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9849        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9850
9851        assert!(matches!(
9852            sleep.as_mut().poll(&mut task_context),
9853            Poll::Pending
9854        ));
9855        assert!(matches!(
9856            sleep.as_mut().poll(&mut task_context),
9857            Poll::Pending
9858        ));
9859
9860        let commands = ctx.take_commands().expect("timer command");
9861        assert_eq!(
9862            commands,
9863            vec![json!({
9864                "type": "start_timer",
9865                "delay_seconds": 2,
9866            })]
9867        );
9868    }
9869
9870    #[test]
9871    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9872        let history = vec![
9873            history_event(
9874                "TimerScheduled",
9875                json!({
9876                    "sequence": 1,
9877                    "timer_id": "timer-1",
9878                    "delay_seconds": 5,
9879                    "fire_at": "2026-07-11T12:00:05Z",
9880                }),
9881            ),
9882            history_event(
9883                "TimerFired",
9884                json!({
9885                    "sequence": 1,
9886                    "timer_id": "timer-1",
9887                    "delay_seconds": 5,
9888                    "fire_at": "2026-07-11T12:00:05Z",
9889                    "fired_at": "2026-07-11T12:00:05Z",
9890                }),
9891            ),
9892        ];
9893
9894        for _restart in 0..2 {
9895            let ctx = workflow_context(history.clone());
9896            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9897            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9898            assert!(matches!(
9899                sleep.as_mut().poll(&mut task_context),
9900                Poll::Ready(Ok(()))
9901            ));
9902            assert!(ctx.take_commands().expect("commands").is_empty());
9903            ctx.ensure_history_consumed().expect("history consumed");
9904        }
9905    }
9906
9907    #[test]
9908    fn workflow_sleep_rejects_changed_delay_during_replay() {
9909        let ctx = workflow_context(vec![
9910            history_event(
9911                "TimerScheduled",
9912                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9913            ),
9914            history_event(
9915                "TimerFired",
9916                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9917            ),
9918        ]);
9919        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
9920        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9921
9922        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9923            sleep.as_mut().poll(&mut task_context)
9924        else {
9925            panic!("changed timer delay must be rejected");
9926        };
9927        assert_eq!(failure.reason, "timer_delay_mismatch");
9928        assert_eq!(failure.sequence, Some(1));
9929    }
9930
9931    #[test]
9932    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
9933        let lone_fire = WorkflowState::new(
9934            vec![history_event(
9935                "TimerFired",
9936                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9937            )],
9938            "rust-workers".to_string(),
9939            JSON_CODEC.to_string(),
9940            None,
9941        )
9942        .expect_err("TimerFired requires TimerScheduled");
9943        assert!(matches!(
9944            lone_fire,
9945            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9946                if reason == "timer_schedule_missing_or_duplicate"
9947        ));
9948
9949        let wrong_identity = WorkflowState::new(
9950            vec![
9951                history_event(
9952                    "TimerScheduled",
9953                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9954                ),
9955                history_event(
9956                    "TimerFired",
9957                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
9958                ),
9959            ],
9960            "rust-workers".to_string(),
9961            JSON_CODEC.to_string(),
9962            None,
9963        )
9964        .expect_err("fire must match scheduled timer identity");
9965        assert!(matches!(
9966            wrong_identity,
9967            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9968                if reason == "timer_identity_mismatch"
9969        ));
9970
9971        let duplicate_fire = WorkflowState::new(
9972            vec![
9973                history_event(
9974                    "TimerScheduled",
9975                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9976                ),
9977                history_event(
9978                    "TimerFired",
9979                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9980                ),
9981                history_event(
9982                    "TimerFired",
9983                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9984                ),
9985            ],
9986            "rust-workers".to_string(),
9987            JSON_CODEC.to_string(),
9988            None,
9989        )
9990        .expect_err("a durable timer cannot fire twice");
9991        assert!(matches!(
9992            duplicate_fire,
9993            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9994                if reason == "duplicate_timer_fire"
9995        ));
9996
9997        let wrong_fired_delay = WorkflowState::new(
9998            vec![
9999                history_event(
10000                    "TimerScheduled",
10001                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10002                ),
10003                history_event(
10004                    "TimerFired",
10005                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10006                ),
10007            ],
10008            "rust-workers".to_string(),
10009            JSON_CODEC.to_string(),
10010            None,
10011        )
10012        .expect_err("timer schedule and fire delays must agree");
10013        assert!(matches!(
10014            wrong_fired_delay,
10015            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10016                if reason == "timer_history_delay_mismatch"
10017        ));
10018    }
10019
10020    #[test]
10021    fn replay_rejects_activity_moved_before_recorded_timer() {
10022        let ctx = workflow_context(vec![
10023            history_event(
10024                "TimerScheduled",
10025                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10026            ),
10027            history_event(
10028                "TimerFired",
10029                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10030            ),
10031            history_event(
10032                "ActivityCompleted",
10033                json!({
10034                    "sequence": 2,
10035                    "activity_type": "after-timer",
10036                    "payload_codec": "json",
10037                    "result": {"codec": "json", "blob": "\"done\""},
10038                }),
10039            ),
10040        ]);
10041        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10042        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10043
10044        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10045            activity.as_mut().poll(&mut task_context)
10046        else {
10047            panic!("reordered durable command must be rejected");
10048        };
10049        assert_eq!(failure.reason, "recorded_command_mismatch");
10050        assert_eq!(failure.sequence, Some(1));
10051        assert_eq!(failure.expected.as_deref(), Some("timer"));
10052        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10053    }
10054
10055    #[test]
10056    fn workflow_context_emits_a_typed_named_signal_wait() {
10057        let ctx = workflow_context(Vec::new());
10058        let mut signal = Box::pin(ctx.wait_signal("finish"));
10059        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10060
10061        assert!(matches!(
10062            signal.as_mut().poll(&mut task_context),
10063            Poll::Pending
10064        ));
10065        assert_eq!(
10066            ctx.take_commands().expect("signal-wait command"),
10067            vec![json!({
10068                "type": "open_signal_wait",
10069                "signal_name": "finish",
10070            })]
10071        );
10072    }
10073
10074    #[test]
10075    fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10076        let ctx = workflow_context(vec![
10077            history_event(
10078                "ConditionWaitOpened",
10079                json!({"sequence": 1, "condition_key": "signal:finish"}),
10080            ),
10081            history_event(
10082                "ConditionWaitSatisfied",
10083                json!({"sequence": 1, "condition_key": "signal:finish"}),
10084            ),
10085            history_event(
10086                "SignalReceived",
10087                json!({"signal_name": "finish", "arguments": []}),
10088            ),
10089        ]);
10090        let mut signal = Box::pin(ctx.wait_signal("finish"));
10091        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10092
10093        assert!(matches!(
10094            signal.as_mut().poll(&mut task_context),
10095            Poll::Pending
10096        ));
10097        assert_eq!(
10098            ctx.take_commands().expect("typed signal-wait command"),
10099            vec![json!({
10100                "type": "open_signal_wait",
10101                "signal_name": "finish",
10102            })]
10103        );
10104    }
10105
10106    #[test]
10107    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10108        let signal_then_timer = vec![
10109            history_event(
10110                "SignalWaitOpened",
10111                json!({"sequence": 1, "signal_name": "go"}),
10112            ),
10113            history_event(
10114                "SignalApplied",
10115                json!({
10116                    "sequence": 1,
10117                    "signal_name": "go",
10118                    "value": {"codec": "json", "blob": "[\"now\"]"},
10119                }),
10120            ),
10121            history_event(
10122                "TimerScheduled",
10123                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10124            ),
10125            history_event(
10126                "TimerFired",
10127                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10128            ),
10129        ];
10130
10131        let ctx = workflow_context(signal_then_timer.clone());
10132        let mut signal = Box::pin(ctx.wait_signal("go"));
10133        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10134        assert!(matches!(
10135            signal.as_mut().poll(&mut task_context),
10136            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10137        ));
10138        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10139        assert!(matches!(
10140            timer.as_mut().poll(&mut task_context),
10141            Poll::Ready(Ok(()))
10142        ));
10143        ctx.ensure_history_consumed()
10144            .expect("signal and timer history consumed in order");
10145
10146        let reordered = workflow_context(signal_then_timer);
10147        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10148        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10149            timer_first.as_mut().poll(&mut task_context)
10150        else {
10151            panic!("timer cannot consume signal-wait-first history");
10152        };
10153        assert_eq!(failure.reason, "recorded_command_mismatch");
10154        assert_eq!(failure.sequence, Some(1));
10155        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10156
10157        let timer_then_signal = vec![
10158            history_event(
10159                "TimerScheduled",
10160                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10161            ),
10162            history_event(
10163                "TimerFired",
10164                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10165            ),
10166            history_event(
10167                "SignalWaitOpened",
10168                json!({"sequence": 2, "signal_name": "go"}),
10169            ),
10170            history_event(
10171                "SignalApplied",
10172                json!({
10173                    "sequence": 2,
10174                    "signal_name": "go",
10175                    "value": {"codec": "json", "blob": "[]"},
10176                }),
10177            ),
10178        ];
10179        let reordered = workflow_context(timer_then_signal);
10180        let mut signal_first = Box::pin(reordered.wait_signal("go"));
10181        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10182            signal_first.as_mut().poll(&mut task_context)
10183        else {
10184            panic!("signal wait cannot consume timer-first history");
10185        };
10186        assert_eq!(failure.reason, "recorded_command_mismatch");
10187        assert_eq!(failure.sequence, Some(1));
10188        assert_eq!(failure.expected.as_deref(), Some("timer"));
10189    }
10190
10191    #[test]
10192    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10193        let duplicate_timer = WorkflowState::new(
10194            vec![
10195                history_event(
10196                    "TimerScheduled",
10197                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10198                ),
10199                history_event(
10200                    "TimerScheduled",
10201                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10202                ),
10203            ],
10204            "rust-workers".to_string(),
10205            JSON_CODEC.to_string(),
10206            None,
10207        )
10208        .expect_err("one workflow sequence cannot schedule two timers");
10209        assert!(matches!(
10210            duplicate_timer,
10211            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10212                if reason == "timer_schedule_missing_or_duplicate"
10213        ));
10214
10215        let colliding_kinds = WorkflowState::new(
10216            vec![
10217                history_event(
10218                    "TimerScheduled",
10219                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10220                ),
10221                history_event(
10222                    "ActivityCompleted",
10223                    json!({"sequence": 1, "activity_type": "same-sequence"}),
10224                ),
10225            ],
10226            "rust-workers".to_string(),
10227            JSON_CODEC.to_string(),
10228            None,
10229        )
10230        .expect_err("one workflow sequence cannot identify two command kinds");
10231        assert!(matches!(
10232            colliding_kinds,
10233            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10234                if reason == "durable_command_sequence_collision"
10235        ));
10236
10237        let duplicate_signal_wait = WorkflowState::new(
10238            vec![
10239                history_event(
10240                    "SignalWaitOpened",
10241                    json!({"sequence": 1, "signal_name": "go"}),
10242                ),
10243                history_event(
10244                    "SignalWaitOpened",
10245                    json!({"sequence": 1, "signal_name": "go"}),
10246                ),
10247            ],
10248            "rust-workers".to_string(),
10249            JSON_CODEC.to_string(),
10250            None,
10251        )
10252        .expect_err("one workflow sequence cannot open two signal waits");
10253        assert!(matches!(
10254            duplicate_signal_wait,
10255            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10256                if reason == "signal_wait_open_missing_or_duplicate"
10257        ));
10258    }
10259
10260    #[test]
10261    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10262        let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10263            .expect("side-effect result");
10264        let ctx = workflow_context(vec![history_event(
10265            "SideEffectRecorded",
10266            json!({"sequence": 99, "result": result}),
10267        )]);
10268
10269        let replayed: Value = ctx
10270            .side_effect(|| panic!("recorded side effect must not run"))
10271            .expect("positive global workflow sequence is valid");
10272        assert_eq!(replayed, json!({"captured": true}));
10273        ctx.ensure_history_consumed().expect("history consumed");
10274    }
10275
10276    #[test]
10277    fn workflow_history_rejects_zero_and_descending_command_sequences() {
10278        let result =
10279            encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10280        let zero = WorkflowState::new(
10281            vec![history_event(
10282                "SideEffectRecorded",
10283                json!({"sequence": 0, "result": result.clone()}),
10284            )],
10285            "rust-workers".to_string(),
10286            JSON_CODEC.to_string(),
10287            None,
10288        )
10289        .expect_err("durable command sequences must be positive");
10290        assert!(matches!(
10291            zero,
10292            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10293                if reason == "durable_command_sequence_invalid"
10294        ));
10295
10296        let descending = WorkflowState::new(
10297            vec![
10298                history_event(
10299                    "SideEffectRecorded",
10300                    json!({"sequence": 3, "result": result}),
10301                ),
10302                history_event(
10303                    "VersionMarkerRecorded",
10304                    json!({
10305                        "sequence": 2,
10306                        "change_id": "descending-marker",
10307                        "version": 1,
10308                        "min_supported": 1,
10309                        "max_supported": 1,
10310                    }),
10311                ),
10312            ],
10313            "rust-workers".to_string(),
10314            JSON_CODEC.to_string(),
10315            None,
10316        )
10317        .expect_err("new durable commands must remain strictly ordered");
10318        let Error::NonDeterministicReplay(failure) = descending else {
10319            panic!("expected typed replay failure");
10320        };
10321        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10322        assert_eq!(failure.sequence, Some(2));
10323        assert_eq!(
10324            failure.expected.as_deref(),
10325            Some("workflow sequence greater than 3")
10326        );
10327        assert_eq!(failure.actual.as_deref(), Some("2"));
10328    }
10329
10330    #[test]
10331    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10332        fn worker() -> Worker {
10333            let client = Client::new("http://127.0.0.1:8080").expect("client");
10334            let mut worker = Worker::new(client, "rust-workers");
10335            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10336                ctx.wait_signal("finish").await?;
10337                let marker: String =
10338                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10339                assert_eq!(marker, "after-finish");
10340                Ok(json!("finished"))
10341            });
10342            worker
10343        }
10344
10345        let marker =
10346            encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10347        let task = workflow_task(
10348            "rust.finish-after-gaps",
10349            vec![
10350                history_event(
10351                    "SignalWaitOpened",
10352                    json!({"sequence": 1, "signal_name": "finish"}),
10353                ),
10354                history_event(
10355                    "SignalReceived",
10356                    json!({
10357                        "signal_id": "increment-3",
10358                        "signal_name": "increment",
10359                        "workflow_sequence": 2,
10360                        "payload_codec": "json",
10361                        "arguments": {"codec": "json", "blob": "[3]"},
10362                    }),
10363                ),
10364                history_event(
10365                    "SignalReceived",
10366                    json!({
10367                        "signal_id": "increment-5",
10368                        "signal_name": "increment",
10369                        "workflow_sequence": 3,
10370                        "payload_codec": "json",
10371                        "arguments": {"codec": "json", "blob": "[5]"},
10372                    }),
10373                ),
10374                history_event(
10375                    "SignalReceived",
10376                    json!({
10377                        "signal_id": "finish",
10378                        "signal_name": "finish",
10379                        "workflow_sequence": 4,
10380                        "payload_codec": "json",
10381                        "arguments": {"codec": "json", "blob": "[]"},
10382                    }),
10383                ),
10384                history_event(
10385                    "SignalApplied",
10386                    json!({
10387                        "sequence": 1,
10388                        "signal_id": "finish",
10389                        "signal_name": "finish",
10390                        "payload_codec": "json",
10391                        "value": {"codec": "json", "blob": "[]"},
10392                    }),
10393                ),
10394                history_event(
10395                    "SideEffectRecorded",
10396                    json!({"sequence": 5, "result": marker}),
10397                ),
10398            ],
10399            JSON_CODEC,
10400        );
10401
10402        for _original_or_cold_worker in 0..2 {
10403            let commands = worker()
10404                .execute_workflow_task(task.clone())
10405                .expect("signal gaps preserve deterministic replay");
10406            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10407            assert_eq!(commands[0]["type"], "complete_workflow");
10408            assert_eq!(
10409                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10410                json!("finished")
10411            );
10412        }
10413    }
10414
10415    #[test]
10416    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10417        let ctx = workflow_context(Vec::new());
10418        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10419        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10420        assert!(matches!(
10421            sleep.as_mut().poll(&mut task_context),
10422            Poll::Ready(Err(Error::TimerDurationOverflow))
10423        ));
10424        assert!(ctx.take_commands().expect("commands").is_empty());
10425    }
10426
10427    #[test]
10428    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10429        let client = Client::new("http://127.0.0.1:8080").expect("client");
10430        let mut worker = Worker::new(client, "rust-workers");
10431        worker.register_workflow("rust.timer", |ctx, _input| async move {
10432            ctx.sleep(Duration::from_secs(5)).await?;
10433            ctx.activity("after-timer", json!([])).await
10434        });
10435
10436        let task = |history_events| WorkflowTask {
10437            task_id: "wft-rust-timer-1".to_string(),
10438            workflow_id: Some("wf-rust-timer".to_string()),
10439            run_id: Some("run-rust-timer".to_string()),
10440            workflow_type: "rust.timer".to_string(),
10441            payload_codec: JSON_CODEC.to_string(),
10442            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10443            history_events,
10444            total_history_events: None,
10445            history_size_bytes: None,
10446            continue_as_new_recommended: None,
10447            history_budget_pressure: None,
10448            next_history_page_token: None,
10449            workflow_task_attempt: 1,
10450            workflow_signal_id: None,
10451            signal_name: None,
10452            signal_arguments: None,
10453            workflow_update_id: None,
10454            update_name: None,
10455            lease_owner: Some("rust-worker".to_string()),
10456        };
10457
10458        let initial = worker
10459            .execute_workflow_task(task(Vec::new()))
10460            .expect("initial timer task");
10461        assert_eq!(
10462            initial,
10463            vec![json!({"type": "start_timer", "delay_seconds": 5})]
10464        );
10465
10466        let replayed = worker
10467            .execute_workflow_task(task(vec![
10468                history_event(
10469                    "TimerScheduled",
10470                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10471                ),
10472                history_event(
10473                    "TimerFired",
10474                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10475                ),
10476                history_event(
10477                    "ActivityCompleted",
10478                    json!({
10479                        "sequence": 2,
10480                        "activity_type": "after-timer",
10481                        "payload_codec": "json",
10482                        "result": {"codec": "json", "blob": "\"done\""},
10483                    }),
10484                ),
10485            ]))
10486            .expect("replayed workflow task");
10487        assert_eq!(replayed.len(), 1);
10488        assert_eq!(replayed[0]["type"], "complete_workflow");
10489        assert_eq!(
10490            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10491            json!("done")
10492        );
10493    }
10494
10495    #[test]
10496    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10497        let client = Client::new("http://127.0.0.1:8080").expect("client");
10498        let mut worker = Worker::new(client, "rust-workers");
10499        worker.register_workflow("rust.continue", |ctx, _input| async move {
10500            ctx.continue_as_new_with_options(
10501                ContinueAsNewOptions::new()
10502                    .workflow_type("rust.next")
10503                    .task_queue("next-workers"),
10504                json!([2, {"cursor": "next"}]),
10505            )
10506        });
10507
10508        let commands = worker
10509            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10510            .expect("continue-as-new command");
10511
10512        assert_eq!(commands.len(), 1);
10513        assert_eq!(commands[0]["type"], "continue_as_new");
10514        assert_eq!(commands[0]["workflow_type"], "rust.next");
10515        assert_eq!(commands[0]["queue"], "next-workers");
10516        assert_eq!(
10517            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10518                .expect("continue-as-new arguments"),
10519            json!([2, {"cursor": "next"}])
10520        );
10521    }
10522
10523    #[test]
10524    fn continue_as_new_preserves_typed_arguments() {
10525        let client = Client::new("http://127.0.0.1:8080").expect("client");
10526        let mut worker = Worker::new(client, "rust-workers");
10527        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10528            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10529            unreachable!("continue-as-new returns a control-flow error")
10530        });
10531
10532        let commands = worker
10533            .execute_workflow_task(workflow_task(
10534                "rust.typed-continue",
10535                Vec::new(),
10536                DEFAULT_CODEC,
10537            ))
10538            .expect("typed continue-as-new command");
10539
10540        assert_eq!(commands[0]["type"], "continue_as_new");
10541        assert_eq!(
10542            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10543                .expect("typed continue arguments"),
10544            AvroValue::Array(vec![typed_fidelity_probe()])
10545        );
10546    }
10547
10548    #[test]
10549    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10550        let client = Client::new("http://127.0.0.1:8080").expect("client");
10551        let mut worker = Worker::new(client, "rust-workers");
10552        worker.register_workflow("rust.continue", |ctx, _input| async move {
10553            ctx.continue_as_new(json!([2]))
10554        });
10555        let task = workflow_task(
10556            "rust.continue",
10557            vec![history_event(
10558                "WorkflowContinuedAsNew",
10559                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10560            )],
10561            JSON_CODEC,
10562        );
10563
10564        for _worker_restart_or_redelivery in 0..2 {
10565            let commands = worker
10566                .execute_workflow_task(task.clone())
10567                .expect("recorded transition replays");
10568            assert!(
10569                commands.is_empty(),
10570                "replay must not emit another successor"
10571            );
10572        }
10573    }
10574
10575    #[test]
10576    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10577        let ctx = workflow_context(Vec::new());
10578        let error = ctx
10579            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
10580            .expect_err("blank queue must be rejected");
10581
10582        let Error::InvalidContinueAsNewOptions(error) = error else {
10583            panic!("expected typed continue-as-new validation error");
10584        };
10585        assert_eq!(error.field, "task_queue");
10586        assert!(ctx.take_commands().expect("commands").is_empty());
10587    }
10588
10589    #[test]
10590    fn workflow_context_exposes_server_history_budget() {
10591        let client = Client::new("http://127.0.0.1:8080").expect("client");
10592        let mut worker = Worker::new(client, "rust-workers");
10593        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10594            let budget = ctx.history_budget()?;
10595            Ok(json!({
10596                "events": budget.event_count,
10597                "bytes": budget.size_bytes,
10598                "recommended": budget.continue_as_new_recommended,
10599                "pressure": budget.pressure,
10600            }))
10601        });
10602        let task: WorkflowTask = serde_json::from_value(json!({
10603            "task_id": "task-history-budget",
10604            "workflow_type": "rust.history-budget",
10605            "payload_codec": JSON_CODEC,
10606            "history_events": [],
10607            "total_history_events": 480,
10608            "history_size_bytes": 1_048_576,
10609            "continue_as_new_recommended": true,
10610            "history_budget_pressure": "continue_as_new_recommended",
10611        }))
10612        .expect("published workflow task");
10613
10614        let commands = worker
10615            .execute_workflow_task(task)
10616            .expect("history-budget workflow");
10617        let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10618        assert_eq!(result["events"], 480);
10619        assert_eq!(result["bytes"], 1_048_576);
10620        assert_eq!(result["recommended"], true);
10621        assert_eq!(result["pressure"], "continue_as_new_recommended");
10622    }
10623
10624    #[test]
10625    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10626        let client = Client::new("http://127.0.0.1:8080").expect("client");
10627        let mut worker = Worker::new(client, "rust-workers");
10628        worker.register_workflow("rust.failing", |_ctx, _input| async move {
10629            Err(Error::Codec("rust_conformance_failure".to_string()))
10630        });
10631        let task = WorkflowTask {
10632            task_id: "wft-rust-failing-1".to_string(),
10633            workflow_id: Some("wf-rust-failing".to_string()),
10634            run_id: Some("run-rust-failing".to_string()),
10635            workflow_type: "rust.failing".to_string(),
10636            payload_codec: JSON_CODEC.to_string(),
10637            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10638            history_events: Vec::new(),
10639            total_history_events: Some(0),
10640            history_size_bytes: None,
10641            continue_as_new_recommended: None,
10642            history_budget_pressure: None,
10643            next_history_page_token: None,
10644            workflow_task_attempt: 1,
10645            workflow_signal_id: None,
10646            signal_name: None,
10647            signal_arguments: None,
10648            workflow_update_id: None,
10649            update_name: None,
10650            lease_owner: Some("rust-worker".to_string()),
10651        };
10652
10653        let commands = worker
10654            .execute_workflow_task(task)
10655            .expect("handler failure becomes a workflow command");
10656
10657        assert_eq!(commands.len(), 1);
10658        assert_eq!(commands[0]["type"], "fail_workflow");
10659        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10660        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10661        assert_eq!(commands[0]["non_retryable"], false);
10662        assert_eq!(
10663            commands[0]["message"],
10664            "codec error: rust_conformance_failure"
10665        );
10666        assert_eq!(
10667            commands[0]["exception"]["message"],
10668            "codec error: rust_conformance_failure"
10669        );
10670    }
10671
10672    #[test]
10673    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10674        let client = Client::new("http://127.0.0.1:8080").expect("client");
10675        let mut worker = Worker::new(client, "rust-workers");
10676        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10677            let _: String = ctx.side_effect(|| "captured".to_string())?;
10678            Err(Error::WorkerLoop("application failure".to_string()))
10679        });
10680
10681        let commands = worker
10682            .execute_workflow_task(workflow_task(
10683                "rust.failing-after-side-effect",
10684                Vec::new(),
10685                JSON_CODEC,
10686            ))
10687            .expect("ordinary failure remains a workflow decision");
10688
10689        assert_eq!(commands.len(), 2);
10690        assert_eq!(commands[0]["type"], "record_side_effect");
10691        assert_eq!(commands[1]["type"], "fail_workflow");
10692    }
10693
10694    #[test]
10695    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10696        let client = Client::new("http://127.0.0.1:8080").expect("client");
10697        let mut worker = Worker::new(client, "rust-workers");
10698        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10699            Err(Error::WorkerLoop("application failure".to_string()))
10700        });
10701        let result =
10702            encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10703
10704        let error = worker
10705            .execute_workflow_task(workflow_task(
10706                "rust.removed-side-effect",
10707                vec![history_event(
10708                    "SideEffectRecorded",
10709                    json!({"sequence": 1, "result": result}),
10710                )],
10711                JSON_CODEC,
10712            ))
10713            .expect_err("removed committed history must not become fail_workflow");
10714
10715        let Error::NonDeterministicReplay(failure) = error else {
10716            panic!("expected typed replay failure");
10717        };
10718        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10719        assert_eq!(failure.sequence, Some(1));
10720        assert_eq!(failure.expected.as_deref(), Some("side effect"));
10721    }
10722
10723    #[test]
10724    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10725        let client = Client::new("http://127.0.0.1:8080").expect("client");
10726        let mut worker = Worker::new(client, "rust-workers");
10727        worker.register_workflow(
10728            "rust.side-effect-before-marker-error",
10729            |ctx, _input| async move {
10730                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10731                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10732                ctx.get_version("restart-safe", 2, 2)?;
10733                Ok(Value::Null)
10734            },
10735        );
10736
10737        let error = worker
10738            .execute_workflow_task(workflow_task(
10739                "rust.side-effect-before-marker-error",
10740                vec![history_event(
10741                    "VersionMarkerRecorded",
10742                    json!({
10743                        "sequence": 1,
10744                        "change_id": "restart-safe",
10745                        "version": 1,
10746                        "min_supported": 1,
10747                        "max_supported": 1,
10748                    }),
10749                )],
10750                JSON_CODEC,
10751            ))
10752            .expect_err("replay error must return no queued workflow commands");
10753
10754        let Error::NonDeterministicReplay(failure) = error else {
10755            panic!("expected typed replay failure");
10756        };
10757        assert_eq!(failure.reason, "version_marker_incompatible_range");
10758        assert_eq!(failure.sequence, Some(1));
10759    }
10760
10761    #[test]
10762    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10763        let client = Client::new("http://127.0.0.1:8080").expect("client");
10764        let mut worker = Worker::new(client, "rust-workers");
10765        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10766            ctx.sleep(Duration::from_secs(5)).await?;
10767            Ok(json!({"status": "timer fired"}))
10768        });
10769
10770        let task = WorkflowTask {
10771            task_id: "wft-rust-timer-pending".to_string(),
10772            workflow_id: Some("wf-rust-timer".to_string()),
10773            run_id: Some("run-rust-timer".to_string()),
10774            workflow_type: "rust.timer.pending".to_string(),
10775            payload_codec: JSON_CODEC.to_string(),
10776            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10777            history_events: vec![history_event(
10778                "TimerScheduled",
10779                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10780            )],
10781            total_history_events: Some(1),
10782            history_size_bytes: None,
10783            continue_as_new_recommended: None,
10784            history_budget_pressure: None,
10785            next_history_page_token: None,
10786            workflow_task_attempt: 1,
10787            workflow_signal_id: None,
10788            signal_name: None,
10789            signal_arguments: None,
10790            workflow_update_id: None,
10791            update_name: None,
10792            lease_owner: Some("rust-worker".to_string()),
10793        };
10794
10795        for _redelivery_or_restart in 0..2 {
10796            let commands = worker
10797                .execute_workflow_task(task.clone())
10798                .expect("recorded timer remains pending");
10799            assert!(
10800                commands.is_empty(),
10801                "recorded timer must not be rescheduled"
10802            );
10803        }
10804    }
10805
10806    #[test]
10807    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10808        let client = Client::new("http://127.0.0.1:8080").expect("client");
10809        let mut worker = Worker::new(client, "rust-workers");
10810        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10811            Ok(json!({"status": "completed"}))
10812        });
10813        let task = WorkflowTask {
10814            task_id: "wft-rust-timer-removed".to_string(),
10815            workflow_id: Some("wf-rust-timer".to_string()),
10816            run_id: Some("run-rust-timer".to_string()),
10817            workflow_type: "rust.timer.removed".to_string(),
10818            payload_codec: JSON_CODEC.to_string(),
10819            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10820            history_events: vec![
10821                history_event(
10822                    "TimerScheduled",
10823                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10824                ),
10825                history_event(
10826                    "TimerFired",
10827                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10828                ),
10829            ],
10830            total_history_events: Some(2),
10831            history_size_bytes: None,
10832            continue_as_new_recommended: None,
10833            history_budget_pressure: None,
10834            next_history_page_token: None,
10835            workflow_task_attempt: 1,
10836            workflow_signal_id: None,
10837            signal_name: None,
10838            signal_arguments: None,
10839            workflow_update_id: None,
10840            update_name: None,
10841            lease_owner: Some("rust-worker".to_string()),
10842        };
10843
10844        let Error::NonDeterministicReplay(failure) = worker
10845            .execute_workflow_task(task)
10846            .expect_err("removed timer must fail replay")
10847        else {
10848            panic!("expected typed replay failure");
10849        };
10850        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10851        assert_eq!(failure.sequence, Some(1));
10852    }
10853
10854    #[test]
10855    fn workflow_context_emits_explicit_child_workflow_contract() {
10856        let ctx = WorkflowContext {
10857            state: Arc::new(Mutex::new(
10858                WorkflowState::new_with_identity(
10859                    Vec::new(),
10860                    Some("wf-parent".to_string()),
10861                    Some("run-parent".to_string()),
10862                    "parent-workers".to_string(),
10863                    JSON_CODEC.to_string(),
10864                    None,
10865                )
10866                .expect("workflow state"),
10867            )),
10868        };
10869        let options = ChildWorkflowOptions::new("python-workers")
10870            .parent_close_policy(ParentClosePolicy::RequestCancel)
10871            .retry_policy(ChildWorkflowRetryPolicy {
10872                max_attempts: Some(3),
10873                backoff_seconds: vec![1, 5],
10874                non_retryable_error_types: vec!["ValidationError".to_string()],
10875            })
10876            .execution_timeout_seconds(600)
10877            .run_timeout_seconds(120);
10878        let mut call = Box::pin(ctx.start_child_workflow(
10879            "python.fulfil-order",
10880            options,
10881            json!([{"order_id": "order-42"}]),
10882        ));
10883        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10884
10885        assert!(matches!(
10886            call.as_mut().poll(&mut task_context),
10887            Poll::Pending
10888        ));
10889        let commands = ctx.take_commands().expect("commands");
10890        assert_eq!(commands.len(), 1);
10891        let command = &commands[0];
10892        assert_eq!(command["type"], "start_child_workflow");
10893        assert_eq!(command["workflow_type"], "python.fulfil-order");
10894        assert_eq!(command["queue"], "python-workers");
10895        assert_eq!(command["parent_close_policy"], "request_cancel");
10896        assert_eq!(command["retry_policy"]["max_attempts"], 3);
10897        assert_eq!(command["execution_timeout_seconds"], 600);
10898        assert_eq!(command["run_timeout_seconds"], 120);
10899        assert_eq!(
10900            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
10901            json!([{"order_id": "order-42"}])
10902        );
10903    }
10904
10905    fn child_parent_worker() -> Worker {
10906        let client = Client::new("http://127.0.0.1:8080").expect("client");
10907        let mut worker = Worker::new(client, "rust-parent-workers");
10908        worker.register_workflow("rust.parent", |ctx, _input| async move {
10909            let child = ctx
10910                .start_child_workflow(
10911                    "python.child",
10912                    ChildWorkflowOptions::new("python-child-workers")
10913                        .parent_close_policy(ParentClosePolicy::Terminate),
10914                    json!([{"codec_probe": [1, true, "rust"]}]),
10915                )
10916                .await?;
10917            Ok(json!({
10918                "parent_workflow_id": child.parent.workflow_id,
10919                "parent_run_id": child.parent.run_id,
10920                "child_workflow_id": child.child.workflow_id,
10921                "child_run_id": child.child.run_id,
10922                "child_workflow_type": child.child_workflow_type,
10923                "result": child.result,
10924            }))
10925        });
10926        worker
10927    }
10928
10929    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
10930        WorkflowTask {
10931            task_id: "wft-child-parent".to_string(),
10932            workflow_id: Some("wf-parent".to_string()),
10933            run_id: Some("run-parent".to_string()),
10934            workflow_type: "rust.parent".to_string(),
10935            payload_codec: JSON_CODEC.to_string(),
10936            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10937            history_events: vec![
10938                HistoryEvent {
10939                    event_type: "ChildWorkflowScheduled".to_string(),
10940                    payload: json!({
10941                        "sequence": 1,
10942                        "child_call_id": "call-child",
10943                        "child_workflow_instance_id": "wf-child",
10944                        "child_workflow_run_id": "run-child",
10945                        "child_workflow_type": "python.child",
10946                    }),
10947                    raw: HashMap::new(),
10948                },
10949                HistoryEvent {
10950                    event_type: event_type.to_string(),
10951                    payload,
10952                    raw: HashMap::new(),
10953                },
10954            ],
10955            total_history_events: Some(2),
10956            history_size_bytes: None,
10957            continue_as_new_recommended: None,
10958            history_budget_pressure: None,
10959            next_history_page_token: None,
10960            workflow_task_attempt: 1,
10961            workflow_signal_id: None,
10962            signal_name: None,
10963            signal_arguments: None,
10964            workflow_update_id: None,
10965            update_name: None,
10966            lease_owner: Some("rust-worker".to_string()),
10967        }
10968    }
10969
10970    #[test]
10971    fn committed_child_result_replays_without_starting_a_duplicate() {
10972        let worker = child_parent_worker();
10973        let task = child_parent_task(
10974            "ChildRunCompleted",
10975            json!({
10976                "sequence": 1,
10977                "child_call_id": "call-child",
10978                "child_workflow_instance_id": "wf-child",
10979                "child_workflow_run_id": "run-child",
10980                "child_workflow_type": "python.child",
10981                "payload_codec": "json",
10982                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
10983            }),
10984        );
10985
10986        for _restart in 0..2 {
10987            let commands = worker
10988                .execute_workflow_task(task.clone())
10989                .expect("replayed parent task");
10990            assert_eq!(commands.len(), 1);
10991            assert_eq!(commands[0]["type"], "complete_workflow");
10992            assert!(!commands
10993                .iter()
10994                .any(|command| command["type"] == "start_child_workflow"));
10995            let output =
10996                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
10997            assert_eq!(output["parent_workflow_id"], "wf-parent");
10998            assert_eq!(output["parent_run_id"], "run-parent");
10999            assert_eq!(output["child_workflow_id"], "wf-child");
11000            assert_eq!(output["child_run_id"], "run-child");
11001            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11002        }
11003    }
11004
11005    #[test]
11006    fn typed_child_arguments_and_results_survive_replay() {
11007        let client = Client::new("http://127.0.0.1:8080").expect("client");
11008        let mut worker = Worker::new(client, "rust-parent-workers");
11009        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11010            let child = ctx
11011                .start_child_workflow_avro_value(
11012                    "python.typed-child",
11013                    ChildWorkflowOptions::new("python-workers"),
11014                    AvroValue::Array(vec![typed_fidelity_probe()]),
11015                )
11016                .await?;
11017            Ok(child.result)
11018        });
11019
11020        let initial = worker
11021            .execute_workflow_task(workflow_task(
11022                "rust.typed-parent",
11023                Vec::new(),
11024                DEFAULT_CODEC,
11025            ))
11026            .expect("typed child start");
11027        assert_eq!(initial[0]["type"], "start_child_workflow");
11028        assert_eq!(
11029            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11030                .expect("typed child arguments"),
11031            AvroValue::Array(vec![typed_fidelity_probe()])
11032        );
11033
11034        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11035            .expect("typed child result");
11036        let task = workflow_task(
11037            "rust.typed-parent",
11038            vec![
11039                history_event(
11040                    "ChildWorkflowScheduled",
11041                    json!({
11042                        "sequence": 1,
11043                        "child_call_id": "call-typed",
11044                        "child_workflow_instance_id": "wf-child",
11045                        "child_workflow_run_id": "run-child",
11046                        "child_workflow_type": "python.typed-child",
11047                    }),
11048                ),
11049                history_event(
11050                    "ChildRunCompleted",
11051                    json!({
11052                        "sequence": 1,
11053                        "child_call_id": "call-typed",
11054                        "child_workflow_instance_id": "wf-child",
11055                        "child_workflow_run_id": "run-child",
11056                        "child_workflow_type": "python.typed-child",
11057                        "payload_codec": DEFAULT_CODEC,
11058                        "result": result,
11059                    }),
11060                ),
11061            ],
11062            DEFAULT_CODEC,
11063        );
11064
11065        let commands = worker
11066            .execute_workflow_task(task)
11067            .expect("typed child replay");
11068        assert_eq!(commands[0]["type"], "complete_workflow");
11069        assert_eq!(
11070            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11071                .expect("typed parent result"),
11072            typed_fidelity_probe()
11073        );
11074    }
11075
11076    #[test]
11077    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11078        let worker = child_parent_worker();
11079        let mut task = child_parent_task("unused", Value::Null);
11080        task.history_events.truncate(1);
11081        task.total_history_events = Some(1);
11082
11083        for _redelivery_or_restart in 0..2 {
11084            let commands = worker
11085                .execute_workflow_task(task.clone())
11086                .expect("recorded child remains pending");
11087            assert!(
11088                commands.is_empty(),
11089                "recorded pending child must not be started again"
11090            );
11091        }
11092    }
11093
11094    #[test]
11095    fn child_cancellation_becomes_stable_parent_failure_command() {
11096        let worker = child_parent_worker();
11097        let task = child_parent_task(
11098            "ChildRunCancelled",
11099            json!({
11100                "sequence": 1,
11101                "child_workflow_instance_id": "wf-child",
11102                "child_workflow_run_id": "run-child",
11103                "child_workflow_type": "python.child",
11104                "failure_id": "failure-child",
11105                "failure_category": "cancelled",
11106                "message": "cancelled by parent-close policy",
11107            }),
11108        );
11109
11110        let commands = worker
11111            .execute_workflow_task(task)
11112            .expect("parent settlement");
11113        assert_eq!(commands.len(), 1);
11114        assert_eq!(commands[0]["type"], "fail_workflow");
11115        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11116        assert_eq!(
11117            commands[0]["exception"]["properties"]["reason"],
11118            "cancelled"
11119        );
11120        assert_eq!(
11121            commands[0]["exception"]["properties"]["child_workflow_run_id"],
11122            "run-child"
11123        );
11124    }
11125
11126    #[test]
11127    fn workflow_can_handle_typed_child_failure() {
11128        let client = Client::new("http://127.0.0.1:8080").expect("client");
11129        let mut worker = Worker::new(client, "rust-parent-workers");
11130        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11131            match ctx
11132                .start_child_workflow(
11133                    "python.child",
11134                    ChildWorkflowOptions::new("python-child-workers"),
11135                    json!([]),
11136                )
11137                .await
11138            {
11139                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11140                    "reason": failure.reason,
11141                    "failure_id": failure.failure_id,
11142                    "exception_class": failure.exception_class,
11143                    "child_run_id": failure.child_workflow_run_id,
11144                })),
11145                Err(error) => Err(error),
11146                Ok(_) => Err(Error::WorkerLoop(
11147                    "child unexpectedly succeeded".to_string(),
11148                )),
11149            }
11150        });
11151        let mut task = child_parent_task(
11152            "ChildRunFailed",
11153            json!({
11154                "sequence": 1,
11155                "child_workflow_instance_id": "wf-child",
11156                "child_workflow_run_id": "run-child",
11157                "child_workflow_type": "python.child",
11158                "failure_id": "failure-child",
11159                "failure_category": "child_workflow",
11160                "message": "payment rejected",
11161                "exception": {
11162                    "type": "PaymentRejected",
11163                    "class": "payments.PaymentRejected",
11164                    "message": "payment rejected"
11165                }
11166            }),
11167        );
11168        task.workflow_type = "rust.handled-parent".to_string();
11169
11170        let commands = worker.execute_workflow_task(task).expect("handled failure");
11171        assert_eq!(commands[0]["type"], "complete_workflow");
11172        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11173        assert_eq!(output["reason"], "child_workflow");
11174        assert_eq!(output["failure_id"], "failure-child");
11175        assert_eq!(output["exception_class"], "payments.PaymentRejected");
11176        assert_eq!(output["child_run_id"], "run-child");
11177    }
11178
11179    #[test]
11180    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11181        let client = Client::new("http://127.0.0.1:8080").expect("client");
11182        let mut worker = Worker::new(client, "rust-workers");
11183
11184        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11185            let signal = ctx.wait_signal("start").await?;
11186            let name = signal
11187                .first()
11188                .and_then(|value| value.as_str())
11189                .unwrap_or("world");
11190            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11191            Ok(json!({
11192                "greeting": greeting,
11193                "language": "rust"
11194            }))
11195        });
11196
11197        let signal_arguments =
11198            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11199        let task = WorkflowTask {
11200            task_id: "wft-rust-signal-1".to_string(),
11201            workflow_id: Some("wf-rust-hello".to_string()),
11202            run_id: Some("run-rust-hello".to_string()),
11203            workflow_type: "rust.hello_workflow".to_string(),
11204            payload_codec: DEFAULT_CODEC.to_string(),
11205            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11206            history_events: vec![HistoryEvent {
11207                event_type: "SignalReceived".to_string(),
11208                payload: json!({
11209                    "signal_id": "sig-rust-1",
11210                    "signal_name": "start"
11211                }),
11212                raw: HashMap::new(),
11213            }],
11214            total_history_events: Some(1),
11215            history_size_bytes: None,
11216            continue_as_new_recommended: None,
11217            history_budget_pressure: None,
11218            next_history_page_token: None,
11219            workflow_task_attempt: 1,
11220            workflow_signal_id: Some("sig-rust-1".to_string()),
11221            signal_name: Some("start".to_string()),
11222            signal_arguments: Some(signal_arguments),
11223            workflow_update_id: None,
11224            update_name: None,
11225            lease_owner: Some("rust-worker".to_string()),
11226        };
11227
11228        let commands = worker.execute_workflow_task(task).expect("workflow task");
11229
11230        assert_eq!(commands.len(), 1);
11231        assert_eq!(commands[0]["type"], "schedule_activity");
11232        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11233        assert_eq!(
11234            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11235            json!(["Rust"])
11236        );
11237    }
11238
11239    #[test]
11240    fn workflow_task_appends_paginated_history_events() {
11241        let mut task = WorkflowTask {
11242            task_id: "wft-rust-pages-1".to_string(),
11243            workflow_id: Some("wf-rust-pages".to_string()),
11244            run_id: Some("run-rust-pages".to_string()),
11245            workflow_type: "rust.hello_workflow".to_string(),
11246            payload_codec: DEFAULT_CODEC.to_string(),
11247            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11248            history_events: vec![HistoryEvent {
11249                event_type: "WorkflowStarted".to_string(),
11250                payload: json!({}),
11251                raw: HashMap::new(),
11252            }],
11253            total_history_events: Some(3),
11254            history_size_bytes: None,
11255            continue_as_new_recommended: None,
11256            history_budget_pressure: None,
11257            next_history_page_token: Some("MQ==".to_string()),
11258            workflow_task_attempt: 1,
11259            workflow_signal_id: None,
11260            signal_name: None,
11261            signal_arguments: None,
11262            workflow_update_id: None,
11263            update_name: None,
11264            lease_owner: Some("rust-worker".to_string()),
11265        };
11266
11267        task.append_history_page(WorkflowTaskHistoryPage {
11268            history_events: vec![
11269                HistoryEvent {
11270                    event_type: "SignalReceived".to_string(),
11271                    payload: json!({
11272                        "signal_id": "sig-rust-1",
11273                        "signal_name": "start",
11274                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11275                            .expect("signal arguments")
11276                    }),
11277                    raw: HashMap::new(),
11278                },
11279                HistoryEvent {
11280                    event_type: "MarkerRecorded".to_string(),
11281                    payload: json!({"sequence": 3}),
11282                    raw: HashMap::new(),
11283                },
11284            ],
11285            total_history_events: Some(3),
11286            next_history_page_token: None,
11287        });
11288
11289        assert_eq!(task.history_events.len(), 3);
11290        assert_eq!(task.total_history_events, Some(3));
11291        assert_eq!(task.next_history_page_token, None);
11292
11293        let signal = task
11294            .history_events
11295            .iter()
11296            .find(|event| event.event_type == "SignalReceived")
11297            .expect("signal event");
11298        assert_eq!(
11299            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11300            vec![AvroValue::String("Rust".to_string())]
11301        );
11302    }
11303
11304    #[tokio::test]
11305    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11306        let client = Client::new("http://127.0.0.1:8080").expect("client");
11307        let mut worker = Worker::new(client, "rust-workers");
11308        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11309        worker.register_query("counter", "current", |ctx, _args| async move {
11310            let mut count = 0_i64;
11311            for signal in ctx.signal_events() {
11312                let value = signal
11313                    .arguments
11314                    .first()
11315                    .and_then(Value::as_i64)
11316                    .unwrap_or_default();
11317                match signal.name.as_str() {
11318                    "increment" => count += value,
11319                    "set" => count = value,
11320                    _ => {}
11321                }
11322            }
11323            Ok(json!(count))
11324        });
11325
11326        let task = QueryTask {
11327            query_task_id: "query-rust-counter".to_string(),
11328            query_task_attempt: 1,
11329            lease_owner: Some("rust-worker".to_string()),
11330            workflow_id: Some("counter-1".to_string()),
11331            run_id: Some("run-counter-1".to_string()),
11332            workflow_type: "counter".to_string(),
11333            query_name: "current".to_string(),
11334            payload_codec: DEFAULT_CODEC.to_string(),
11335            workflow_arguments: Some(
11336                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11337            ),
11338            query_arguments: Some(
11339                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11340            ),
11341            history_events: vec![
11342                HistoryEvent {
11343                    event_type: "SignalReceived".to_string(),
11344                    payload: json!({
11345                        "signal_id": "php-signal-1",
11346                        "signal_name": "increment",
11347                        "workflow_sequence": 1,
11348                        "payload_codec": DEFAULT_CODEC,
11349                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11350                    }),
11351                    raw: HashMap::new(),
11352                },
11353                HistoryEvent {
11354                    event_type: "SignalReceived".to_string(),
11355                    payload: json!({
11356                        "signal_id": "python-signal-2",
11357                        "signal_name": "increment",
11358                        "workflow_sequence": 2,
11359                        "payload_codec": JSON_CODEC,
11360                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11361                    }),
11362                    raw: HashMap::new(),
11363                },
11364                HistoryEvent {
11365                    event_type: "SignalReceived".to_string(),
11366                    payload: json!({
11367                        "signal_id": "rust-signal-3",
11368                        "signal_name": "set",
11369                        "workflow_sequence": 3,
11370                        "payload_codec": DEFAULT_CODEC,
11371                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11372                    }),
11373                    raw: HashMap::new(),
11374                },
11375            ],
11376            history_export: None,
11377            run_status: Some("completed".to_string()),
11378        };
11379
11380        let result = worker.execute_query_task(task).await.expect("query result");
11381        assert_eq!(result.into_json().expect("query projection"), json!(0));
11382    }
11383
11384    #[tokio::test]
11385    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11386        let worker = replay_counter_worker();
11387        let running_history = json!([
11388            {
11389                "type": "ActivityCompleted",
11390                "payload": {
11391                    "sequence": 1,
11392                    "activity_type": "load-counter",
11393                    "payload_codec": "json",
11394                    "result": {"codec": "json", "blob": "\"loaded\""}
11395                }
11396            },
11397            {
11398                "type": "SignalWaitOpened",
11399                "payload": {
11400                    "sequence": 3,
11401                    "signal_name": "increment"
11402                }
11403            },
11404            {
11405                "type": "SignalReceived",
11406                "payload": {
11407                    "signal_id": "signal-3",
11408                    "signal_name": "increment",
11409                    "workflow_sequence": 2,
11410                    "payload_codec": "json",
11411                    "arguments": {"codec": "json", "blob": "[3]"}
11412                }
11413            },
11414            {
11415                "type": "SignalApplied",
11416                "payload": {
11417                    "sequence": 3,
11418                    "signal_id": "signal-3",
11419                    "signal_name": "increment",
11420                    "payload_codec": "json",
11421                    "value": {"codec": "json", "blob": "[3]"}
11422                }
11423            }
11424        ]);
11425
11426        let running = worker
11427            .execute_query_task(replay_counter_query(
11428                "current",
11429                running_history.clone(),
11430                "running",
11431            ))
11432            .await
11433            .expect("running replay query");
11434        assert_eq!(
11435            running.clone().into_json().expect("query projection"),
11436            json!({"loaded": "loaded", "count": 3, "finished": false})
11437        );
11438
11439        let detached = worker
11440            .execute_query_task(replay_counter_query(
11441                "detached-mutation",
11442                running_history.clone(),
11443                "running",
11444            ))
11445            .await
11446            .expect("query mutates only its detached state clone");
11447        assert_eq!(detached.into_json().expect("query projection"), json!(999));
11448        let failed = worker
11449            .execute_query_task(replay_counter_query(
11450                "failed-mutation",
11451                running_history.clone(),
11452                "running",
11453            ))
11454            .await
11455            .expect_err("failed query");
11456        assert_eq!(failed.reason, "query_rejected");
11457        let unchanged = worker
11458            .execute_query_task(replay_counter_query("current", running_history, "running"))
11459            .await
11460            .expect("later query reconstructs unchanged state");
11461        assert_eq!(unchanged, running);
11462
11463        let restarted_worker = replay_counter_worker();
11464        let restarted_task: QueryTask = serde_json::from_value(json!({
11465            "query_task_id": "query-after-restart",
11466            "workflow_id": "counter-1",
11467            "run_id": "run-counter-1",
11468            "workflow_type": "replay-counter",
11469            "query_name": "current",
11470            "payload_codec": "json",
11471            "workflow_arguments": {"codec": "json", "blob": "[]"},
11472            "query_arguments": {"codec": "json", "blob": "[]"},
11473            "history_events": [],
11474            "history_export": {
11475                "payloads": {"codec": "json"},
11476                "history_events": [
11477                    {
11478                        "type": "ActivityCompleted",
11479                        "payload": {
11480                            "sequence": 1,
11481                            "activity_type": "load-counter",
11482                            "payload_codec": "json",
11483                            "result": null
11484                        }
11485                    },
11486                    {
11487                        "type": "SignalWaitOpened",
11488                        "payload": {
11489                            "sequence": 3,
11490                            "signal_name": "increment"
11491                        }
11492                    },
11493                    {
11494                        "type": "SignalReceived",
11495                        "payload": {
11496                            "signal_id": "signal-3",
11497                            "signal_name": "increment",
11498                            "workflow_sequence": 2
11499                        }
11500                    },
11501                    {
11502                        "type": "SignalApplied",
11503                        "payload": {
11504                            "sequence": 3,
11505                            "signal_id": "signal-3",
11506                            "signal_name": "increment"
11507                        }
11508                    },
11509                    {
11510                        "type": "SignalWaitOpened",
11511                        "payload": {
11512                            "sequence": 5,
11513                            "signal_name": "increment"
11514                        }
11515                    },
11516                    {
11517                        "type": "SignalReceived",
11518                        "payload": {
11519                            "signal_id": "signal-5",
11520                            "signal_name": "increment",
11521                            "workflow_sequence": 4
11522                        }
11523                    },
11524                    {
11525                        "type": "SignalApplied",
11526                        "payload": {
11527                            "sequence": 5,
11528                            "signal_id": "signal-5",
11529                            "signal_name": "increment"
11530                        }
11531                    }
11532                ],
11533                "activities": [{
11534                    "sequence": 1,
11535                    "activity_type": "load-counter",
11536                    "payload_codec": "json",
11537                    "result": {"codec": "json", "blob": "\"loaded\""}
11538                }],
11539                "signals": [
11540                    {
11541                        "id": "signal-3",
11542                        "name": "increment",
11543                        "workflow_sequence": 2,
11544                        "payload_codec": "json",
11545                        "arguments": "[3]"
11546                    },
11547                    {
11548                        "id": "signal-5",
11549                        "name": "increment",
11550                        "workflow_sequence": 4,
11551                        "payload_codec": "json",
11552                        "arguments": "[5]"
11553                    }
11554                ]
11555            },
11556            "run_status": "completed"
11557        }))
11558        .expect("cold replay query task");
11559        let completed = restarted_worker
11560            .execute_query_task(restarted_task)
11561            .await
11562            .expect("completed cold replay query");
11563        assert_eq!(
11564            completed.into_json().expect("query projection"),
11565            json!({"loaded": "loaded", "count": 8, "finished": true})
11566        );
11567    }
11568
11569    #[tokio::test]
11570    async fn replayed_query_replay_failures_are_machine_readable() {
11571        let worker = replay_counter_worker();
11572        let task = replay_counter_query(
11573            "current",
11574            json!([{
11575                "type": "ActivityCompleted",
11576                "payload": {
11577                    "sequence": 1,
11578                    "payload_codec": "json",
11579                    "result": {"codec": "json", "blob": "{"}
11580                }
11581            }]),
11582            "running",
11583        );
11584        let failure = worker
11585            .execute_query_task(task)
11586            .await
11587            .expect_err("invalid replay history payload");
11588        assert_eq!(failure.reason, "query_workflow_state_unavailable");
11589        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11590    }
11591
11592    #[tokio::test]
11593    async fn query_task_restores_compact_history_from_export() {
11594        let client = Client::new("http://127.0.0.1:8080").expect("client");
11595        let mut worker = Worker::new(client, "rust-workers");
11596        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11597        worker.register_query("counter", "current", |ctx, _args| async move {
11598            Ok(json!(ctx.signals("increment")[0][0]))
11599        });
11600        let task: QueryTask = serde_json::from_value(json!({
11601            "query_task_id": "query-export",
11602            "workflow_type": "counter",
11603            "query_name": "current",
11604            "payload_codec": "json",
11605            "workflow_arguments": {"codec": "json", "blob": "[]"},
11606            "query_arguments": {"codec": "json", "blob": "[]"},
11607            "history_events": [],
11608            "history_export": {
11609                "payloads": {"codec": "json"},
11610                "history_events": [{
11611                    "type": "SignalReceived",
11612                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11613                }],
11614                "signals": [{
11615                    "id": "signal-export",
11616                    "name": "increment",
11617                    "status": "applied",
11618                    "workflow_sequence": 1,
11619                    "payload_codec": "json",
11620                    "arguments": "[9]"
11621                }]
11622            }
11623        }))
11624        .expect("query task");
11625
11626        let result = worker.execute_query_task(task).await.expect("query result");
11627        assert_eq!(result.into_json().expect("query projection"), json!(9));
11628    }
11629
11630    #[tokio::test]
11631    async fn query_task_failures_have_stable_reasons() {
11632        let client = Client::new("http://127.0.0.1:8080").expect("client");
11633        let mut worker = Worker::new(client, "rust-workers");
11634        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11635        worker.register_query(
11636            "counter",
11637            "current",
11638            |_ctx, _args| async move { Ok(json!(0)) },
11639        );
11640
11641        let base_task = QueryTask {
11642            query_task_id: "query-errors".to_string(),
11643            query_task_attempt: 1,
11644            lease_owner: None,
11645            workflow_id: Some("counter-errors".to_string()),
11646            run_id: Some("run-errors".to_string()),
11647            workflow_type: "counter".to_string(),
11648            query_name: "missing".to_string(),
11649            payload_codec: JSON_CODEC.to_string(),
11650            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11651            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11652            history_events: Vec::new(),
11653            history_export: None,
11654            run_status: Some("running".to_string()),
11655        };
11656
11657        let unknown = worker
11658            .execute_query_task(base_task.clone())
11659            .await
11660            .expect_err("unknown query");
11661        assert_eq!(unknown.reason, "rejected_unknown_query");
11662
11663        let mut malformed = base_task;
11664        malformed.query_name = "current".to_string();
11665        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11666        let malformed = worker
11667            .execute_query_task(malformed)
11668            .await
11669            .expect_err("malformed payload");
11670        assert_eq!(malformed.reason, "query_payload_decode_failed");
11671
11672        let client = Client::new("http://127.0.0.1:8080").expect("client");
11673        let mut unavailable_worker = Worker::new(client, "rust-workers");
11674        unavailable_worker
11675            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11676        let unavailable_task: QueryTask = serde_json::from_value(json!({
11677            "query_task_id": "query-unavailable",
11678            "workflow_type": "counter",
11679            "query_name": "current",
11680            "payload_codec": "json",
11681            "workflow_arguments": {"codec": "json", "blob": "[]"},
11682            "query_arguments": {"codec": "json", "blob": "[]"}
11683        }))
11684        .expect("query task");
11685        let unavailable = unavailable_worker
11686            .execute_query_task(unavailable_task)
11687            .await
11688            .expect_err("query handler unavailable");
11689        assert_eq!(unavailable.reason, "query_handler_unavailable");
11690    }
11691
11692    #[tokio::test]
11693    async fn client_query_decodes_result_and_typed_failure() {
11694        let server = MockWorkerServer::start();
11695        let client = Client::builder(server.base_url())
11696            .timeout(Duration::from_secs(2))
11697            .build()
11698            .expect("client");
11699
11700        let result = client
11701            .query_workflow("counter-1", "current", json!([]))
11702            .await
11703            .expect("query result");
11704        assert_eq!(result, json!({"count": 8}));
11705
11706        let error = client
11707            .query_workflow("counter-1", "missing", json!([]))
11708            .await
11709            .expect_err("unknown query");
11710        let Error::QueryFailed(failure) = error else {
11711            panic!("expected typed query failure");
11712        };
11713        assert_eq!(failure.status, 404);
11714        assert_eq!(failure.reason, "rejected_unknown_query");
11715    }
11716
11717    #[tokio::test]
11718    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11719        let server = MockWorkerServer::start();
11720        let client = Client::builder(server.base_url())
11721            .timeout(Duration::from_secs(2))
11722            .build()
11723            .expect("client");
11724        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11725
11726        client
11727            .start_workflow(
11728                "typed.echo",
11729                "rust-workers",
11730                "typed-start",
11731                arguments.clone(),
11732            )
11733            .await
11734            .expect("typed workflow start");
11735        assert_eq!(
11736            decode_wire_avro_value(
11737                &server.request_body("/api/workflows")["input"],
11738                DEFAULT_CODEC,
11739            )
11740            .expect("typed start input"),
11741            arguments
11742        );
11743
11744        client
11745            .signal_workflow("typed-1", "changed", arguments.clone())
11746            .await
11747            .expect("typed signal");
11748        assert_eq!(
11749            decode_wire_avro_value(
11750                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11751                DEFAULT_CODEC,
11752            )
11753            .expect("typed signal input"),
11754            arguments
11755        );
11756
11757        assert_eq!(
11758            client
11759                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11760                .await
11761                .expect("typed query"),
11762            typed_fidelity_probe()
11763        );
11764        assert_eq!(
11765            decode_wire_avro_value(
11766                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11767                DEFAULT_CODEC,
11768            )
11769            .expect("typed query input"),
11770            arguments
11771        );
11772
11773        assert_eq!(
11774            client
11775                .update_workflow_avro_value(
11776                    "typed-1",
11777                    "replace",
11778                    arguments.clone(),
11779                    Some("typed-request"),
11780                )
11781                .await
11782                .expect("typed update"),
11783            typed_fidelity_probe()
11784        );
11785        let update = server.request_body("/api/workflows/typed-1/update/replace");
11786        assert_eq!(update["request_id"], "typed-request");
11787        assert_eq!(
11788            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11789            arguments
11790        );
11791
11792        let handle = WorkflowHandle {
11793            client: client.clone(),
11794            workflow_id: "typed-1".to_string(),
11795            run_id: Some("run-typed-1".to_string()),
11796            workflow_type: "typed.echo".to_string(),
11797        };
11798        assert_eq!(
11799            handle
11800                .result_avro_value(WorkflowResultOptions::default())
11801                .await
11802                .expect("typed workflow result"),
11803            typed_fidelity_probe()
11804        );
11805
11806        client
11807            .complete_activity_task(
11808                "activity-typed",
11809                "attempt-typed",
11810                "rust-worker",
11811                typed_fidelity_probe(),
11812                DEFAULT_CODEC,
11813            )
11814            .await
11815            .expect("typed activity completion");
11816        assert_eq!(
11817            decode_wire_avro_value(
11818                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11819                    ["result"],
11820                DEFAULT_CODEC,
11821            )
11822            .expect("typed activity result"),
11823            typed_fidelity_probe()
11824        );
11825        client
11826            .fail_activity_task(
11827                "activity-typed",
11828                "attempt-typed",
11829                "rust-worker",
11830                "typed failure",
11831                true,
11832            )
11833            .await
11834            .expect("activity failure");
11835    }
11836
11837    #[tokio::test]
11838    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11839        let server = MockWorkerServer::start();
11840        let client = Client::builder(server.base_url())
11841            .timeout(Duration::from_secs(2))
11842            .build()
11843            .expect("client");
11844
11845        let options = WorkflowCommandOptions::new()
11846            .reason("cleanup requested")
11847            .request_id("cancel-17");
11848        let cancelled = client
11849            .cancel_workflow("wf-lifecycle", options)
11850            .await
11851            .expect("instance cancellation");
11852        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11853        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11854        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11855        assert_eq!(
11856            server.request_body("/api/workflows/wf-lifecycle/cancel"),
11857            json!({"reason":"cleanup requested","request_id":"cancel-17"})
11858        );
11859
11860        let terminated = client
11861            .terminate_workflow(
11862                "wf-lifecycle",
11863                WorkflowCommandOptions::new().reason("forced stop"),
11864            )
11865            .await
11866            .expect("instance termination");
11867        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11868        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11869
11870        client
11871            .cancel_workflow_run(
11872                "wf-lifecycle",
11873                "run-current",
11874                WorkflowCommandOptions::default(),
11875            )
11876            .await
11877            .expect("selected run cancellation");
11878        client
11879            .terminate_workflow_run(
11880                "wf-lifecycle",
11881                "run-current",
11882                WorkflowCommandOptions::default(),
11883            )
11884            .await
11885            .expect("selected run termination");
11886
11887        for (command, error) in [
11888            (
11889                WorkflowCommandKind::Cancel,
11890                client
11891                    .cancel_workflow_run(
11892                        "wf-lifecycle",
11893                        "run-stale",
11894                        WorkflowCommandOptions::default(),
11895                    )
11896                    .await
11897                    .expect_err("stale cancellation must be rejected"),
11898            ),
11899            (
11900                WorkflowCommandKind::Terminate,
11901                client
11902                    .terminate_workflow_run(
11903                        "wf-lifecycle",
11904                        "run-stale",
11905                        WorkflowCommandOptions::default(),
11906                    )
11907                    .await
11908                    .expect_err("stale termination must be rejected"),
11909            ),
11910        ] {
11911            let Error::WorkflowCommandRejected(rejection) = error else {
11912                panic!("expected typed command rejection");
11913            };
11914            assert_eq!(rejection.command, command);
11915            assert_eq!(rejection.status, 409);
11916            assert_eq!(rejection.reason, "historical_run_command_rejected");
11917            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
11918            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
11919        }
11920    }
11921
11922    #[tokio::test]
11923    async fn workflow_start_options_send_server_enforced_deadlines() {
11924        let server = MockWorkerServer::start();
11925        let client = Client::builder(server.base_url())
11926            .timeout(Duration::from_secs(2))
11927            .build()
11928            .expect("client");
11929
11930        let handle = client
11931            .start_workflow_with_options(
11932                "rust.timeout",
11933                "rust-timeouts",
11934                "wf-start-options",
11935                WorkflowStartOptions::new()
11936                    .execution_timeout_seconds(30)
11937                    .run_timeout_seconds(1),
11938                json!([]),
11939            )
11940            .await
11941            .expect("workflow start");
11942
11943        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
11944        let body = server.request_body("/api/workflows");
11945        assert_eq!(body["execution_timeout_seconds"], 30);
11946        assert_eq!(body["run_timeout_seconds"], 1);
11947
11948        let invalid = client
11949            .start_workflow_with_options(
11950                "rust.timeout",
11951                "rust-timeouts",
11952                "wf-invalid-options",
11953                WorkflowStartOptions::new()
11954                    .execution_timeout_seconds(1)
11955                    .run_timeout_seconds(2),
11956                json!([]),
11957            )
11958            .await
11959            .expect_err("invalid deadline ordering");
11960        assert!(invalid
11961            .to_string()
11962            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
11963    }
11964
11965    #[tokio::test]
11966    async fn workflow_result_returns_each_typed_terminal_outcome() {
11967        let server = MockWorkerServer::start();
11968        let client = Client::builder(server.base_url())
11969            .timeout(Duration::from_secs(2))
11970            .build()
11971            .expect("client");
11972        let options = WorkflowResultOptions {
11973            poll_interval: Duration::ZERO,
11974            timeout: Duration::from_secs(1),
11975        };
11976
11977        let failed = WorkflowHandle {
11978            client: client.clone(),
11979            workflow_id: "wf-failed".to_string(),
11980            run_id: Some("run-failed".to_string()),
11981            workflow_type: "failure".to_string(),
11982        }
11983        .result(options)
11984        .await
11985        .expect_err("failed outcome");
11986        let Error::WorkflowFailed(failure) = failed else {
11987            panic!("expected WorkflowFailed");
11988        };
11989        assert_eq!(failure.workflow_id, "wf-failed");
11990        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
11991        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
11992        assert_eq!(failure.failure_category.as_deref(), Some("application"));
11993        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
11994        assert_eq!(
11995            failure.exception_class.as_deref(),
11996            Some("billing::PaymentError")
11997        );
11998        assert_eq!(failure.non_retryable, Some(true));
11999
12000        for (workflow_id, expected_kind, expected_reason) in [
12001            (
12002                "wf-cancelled",
12003                WorkflowTerminalKind::Cancelled,
12004                "cleanup requested",
12005            ),
12006            (
12007                "wf-terminated",
12008                WorkflowTerminalKind::Terminated,
12009                "forced stop",
12010            ),
12011            (
12012                "wf-timed-out",
12013                WorkflowTerminalKind::TimedOut,
12014                "run_timeout",
12015            ),
12016        ] {
12017            let error = WorkflowHandle {
12018                client: client.clone(),
12019                workflow_id: workflow_id.to_string(),
12020                run_id: None,
12021                workflow_type: "terminal".to_string(),
12022            }
12023            .result(options)
12024            .await
12025            .expect_err("typed terminal outcome");
12026            let outcome = match error {
12027                Error::WorkflowCancelled(outcome) => outcome,
12028                Error::WorkflowTerminated(outcome) => outcome,
12029                Error::WorkflowTimedOut(outcome) => outcome,
12030                other => panic!("unexpected terminal error: {other}"),
12031            };
12032            assert_eq!(outcome.kind, expected_kind);
12033            assert_eq!(outcome.workflow_id, workflow_id);
12034            assert_eq!(outcome.reason, expected_reason);
12035        }
12036
12037        let wait_timeout = WorkflowHandle {
12038            client,
12039            workflow_id: "wf-waiting".to_string(),
12040            run_id: Some("run-waiting".to_string()),
12041            workflow_type: "waiting".to_string(),
12042        }
12043        .result(WorkflowResultOptions {
12044            poll_interval: Duration::ZERO,
12045            timeout: Duration::ZERO,
12046        })
12047        .await
12048        .expect_err("client wait timeout");
12049        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12050            panic!("expected typed client timeout");
12051        };
12052        assert_eq!(timeout.reason, "result_wait_timeout");
12053        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12054        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12055    }
12056
12057    #[tokio::test]
12058    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12059        let server = MockWorkerServer::start();
12060        let client = Client::builder(server.base_url())
12061            .timeout(Duration::from_secs(2))
12062            .build()
12063            .expect("client");
12064
12065        let handle = WorkflowHandle {
12066            client,
12067            workflow_id: "wf-selected".to_string(),
12068            run_id: Some("run-selected".to_string()),
12069            workflow_type: "selected".to_string(),
12070        };
12071        let options = WorkflowResultOptions {
12072            poll_interval: Duration::ZERO,
12073            timeout: Duration::from_secs(1),
12074        };
12075
12076        let current = handle
12077            .result(options)
12078            .await
12079            .expect("instance result follows the current run");
12080        assert_eq!(current, json!("current run output"));
12081
12082        let error = handle
12083            .result_selected_run(options)
12084            .await
12085            .expect_err("the selected run is cancelled even though the current run completed");
12086
12087        let Error::WorkflowCancelled(outcome) = error else {
12088            panic!("expected selected run cancellation");
12089        };
12090        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12091        assert_eq!(outcome.reason, "selected run cancelled");
12092        assert_eq!(
12093            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12094            1
12095        );
12096        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12097    }
12098
12099    #[tokio::test]
12100    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12101        let server = MockWorkerServer::draining_polls();
12102        let client = Client::builder(server.base_url())
12103            .timeout(Duration::from_secs(2))
12104            .build()
12105            .expect("client");
12106
12107        let workflow = client
12108            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12109            .await
12110            .expect("workflow drain response");
12111        let activity = client
12112            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12113            .await
12114            .expect("activity drain response");
12115        let query = client
12116            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12117            .await
12118            .expect("query drain response");
12119
12120        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12121            assert_eq!(
12122                outcome,
12123                WorkerPollOutcome::Stop {
12124                    poll_status: Some("draining".to_string()),
12125                    reason: Some("worker_draining".to_string()),
12126                }
12127            );
12128        }
12129
12130        assert!(client
12131            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12132            .await
12133            .expect("compatibility poll")
12134            .is_none());
12135    }
12136
12137    #[tokio::test]
12138    async fn managed_worker_honors_drain_stop_for_every_task_family() {
12139        let server = MockWorkerServer::draining_polls();
12140        let client = Client::builder(server.base_url())
12141            .timeout(Duration::from_secs(2))
12142            .build()
12143            .expect("client");
12144
12145        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12146            .worker_id("draining-workflow-worker")
12147            .poll_timeout(Duration::ZERO);
12148        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12149        workflow_worker
12150            .run()
12151            .await
12152            .expect("workflow drain is a clean stop");
12153
12154        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12155            .worker_id("draining-activity-worker")
12156            .poll_timeout(Duration::ZERO);
12157        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12158        activity_worker
12159            .run()
12160            .await
12161            .expect("activity drain is a clean stop");
12162
12163        let mut query_worker = Worker::new(client, "rust-workers")
12164            .worker_id("draining-query-worker")
12165            .poll_timeout(Duration::ZERO);
12166        query_worker.register_query("counter", "current", |_ctx, _args| async {
12167            Ok(Value::Null)
12168        });
12169        query_worker
12170            .run()
12171            .await
12172            .expect("query drain is a clean stop");
12173    }
12174
12175    #[tokio::test]
12176    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12177        let server = MockWorkerServer::start();
12178        let client = Client::builder(server.base_url())
12179            .timeout(Duration::from_secs(2))
12180            .build()
12181            .expect("client");
12182
12183        let heartbeat = client
12184            .heartbeat_activity_task(
12185                "activity-cancel",
12186                "attempt-cancel",
12187                "rust-worker",
12188                typed_fidelity_probe(),
12189            )
12190            .await
12191            .expect("cancellation heartbeat");
12192        assert!(heartbeat.cancel_requested);
12193        assert!(heartbeat.should_stop());
12194        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12195        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12196        let heartbeat_body =
12197            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12198        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12199        assert_eq!(
12200            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12201                .expect("typed heartbeat details"),
12202            typed_fidelity_probe()
12203        );
12204
12205        let error = client
12206            .complete_activity_task(
12207                "activity-cancel",
12208                "attempt-cancel",
12209                "rust-worker",
12210                json!({"late":true}),
12211                JSON_CODEC,
12212            )
12213            .await
12214            .expect_err("late completion must be refused");
12215        assert!(activity_task_rejection_is_final(&error));
12216        let Error::ActivityTaskRejected(rejection) = error else {
12217            panic!("expected typed activity rejection");
12218        };
12219        assert_eq!(rejection.status, 409);
12220        assert_eq!(rejection.reason, "run_cancelled");
12221        assert!(rejection.cancel_requested);
12222        assert_eq!(rejection.can_continue, Some(false));
12223    }
12224
12225    #[tokio::test]
12226    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12227        let server = MockWorkerServer::cancelled_activity();
12228        let client = Client::builder(server.base_url())
12229            .timeout(Duration::from_secs(2))
12230            .build()
12231            .expect("client");
12232        let cancellation_observed = Arc::new(AtomicBool::new(false));
12233        let observed = Arc::clone(&cancellation_observed);
12234        let mut worker = Worker::new(client.clone(), "rust-workers")
12235            .worker_id("rust-cancel-worker")
12236            .poll_timeout(Duration::from_millis(10));
12237        worker.register_activity("cancel-aware", move |ctx, _args| {
12238            let observed = Arc::clone(&observed);
12239            async move {
12240                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12241                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12242                Ok(json!({"late":"completion"}))
12243            }
12244        });
12245
12246        assert_eq!(
12247            worker.run_once().await.expect("cancelled attempt handled"),
12248            1
12249        );
12250        assert!(cancellation_observed.load(Ordering::SeqCst));
12251        assert_eq!(
12252            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12253            1
12254        );
12255
12256        let mut restarted = Worker::new(client, "rust-workers")
12257            .worker_id("rust-cancel-worker-restarted")
12258            .poll_timeout(Duration::from_millis(10));
12259        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12260        assert_eq!(
12261            restarted
12262                .run_once()
12263                .await
12264                .expect("replacement worker continues polling"),
12265            0
12266        );
12267    }
12268
12269    #[tokio::test]
12270    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12271        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12272        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12273        let client = Client::builder(server.base_url())
12274            .timeout(Duration::from_secs(2))
12275            .build()
12276            .expect("client");
12277
12278        let direct_error = client
12279            .complete_workflow_task(
12280                "workflow-timeout-task",
12281                "timeout-worker",
12282                3,
12283                vec![json!({"type": "complete_workflow", "result": null})],
12284            )
12285            .await
12286            .expect_err("the low-level client preserves the completion rejection");
12287        let Error::Http { status, body } = direct_error else {
12288            panic!("expected the original HTTP completion rejection");
12289        };
12290        assert_eq!(status, reqwest::StatusCode::CONFLICT);
12291        assert_eq!(
12292            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12293            "run_timed_out"
12294        );
12295
12296        let mut worker = Worker::new(client, "rust-workers")
12297            .worker_id("timeout-worker")
12298            .poll_timeout(Duration::from_millis(10));
12299        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12300            Ok(json!({"late": "result"}))
12301        });
12302
12303        assert_eq!(
12304            worker
12305                .run_once()
12306                .await
12307                .expect("authoritative selected-run timeout settles the tick"),
12308            1
12309        );
12310        assert_eq!(
12311            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12312            2,
12313            "both the direct client proof and managed worker must see the rejection"
12314        );
12315    }
12316
12317    #[tokio::test]
12318    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12319        for (name, status, response) in [
12320            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12321            (
12322                "command was recorded",
12323                "409 Conflict",
12324                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12325            ),
12326            (
12327                "lease conflict",
12328                "409 Conflict",
12329                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12330            ),
12331            (
12332                "nonterminal run",
12333                "409 Conflict",
12334                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12335            ),
12336            (
12337                "different selected run",
12338                "409 Conflict",
12339                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12340            ),
12341            (
12342                "different task attempt",
12343                "409 Conflict",
12344                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12345            ),
12346            (
12347                "authentication failure",
12348                "401 Unauthorized",
12349                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12350            ),
12351            (
12352                "authorization failure",
12353                "403 Forbidden",
12354                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12355            ),
12356            (
12357                "protocol failure",
12358                "400 Bad Request",
12359                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12360            ),
12361            (
12362                "malformed command",
12363                "422 Unprocessable Entity",
12364                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12365            ),
12366            (
12367                "transient server failure",
12368                "503 Service Unavailable",
12369                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12370            ),
12371        ] {
12372            let server = MockWorkerServer::workflow_completion(status, response);
12373            let client = Client::builder(server.base_url())
12374                .timeout(Duration::from_secs(2))
12375                .build()
12376                .expect("client");
12377            let mut worker = Worker::new(client, "rust-workers")
12378                .worker_id("timeout-worker")
12379                .poll_timeout(Duration::from_millis(10));
12380            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12381                Ok(json!({"late": "result"}))
12382            });
12383
12384            let error = worker
12385                .run_once()
12386                .await
12387                .expect_err(&format!("{name} must remain an error"));
12388            assert!(
12389                matches!(error, Error::Http { .. } | Error::Protocol(_)),
12390                "{name} returned an unexpected error variant: {error}"
12391            );
12392        }
12393    }
12394
12395    #[tokio::test]
12396    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12397        let server = MockWorkerServer::start();
12398        let client = Client::builder(server.base_url())
12399            .timeout(Duration::from_secs(2))
12400            .build()
12401            .expect("client");
12402
12403        client
12404            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12405            .await
12406            .expect("register");
12407        client
12408            .heartbeat_worker("capture-worker", 1, 1)
12409            .await
12410            .expect("heartbeat");
12411        client
12412            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12413            .await
12414            .expect("workflow poll");
12415        client
12416            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12417            .await
12418            .expect("activity poll");
12419
12420        for path in [
12421            "/api/worker/register",
12422            "/api/worker/heartbeat",
12423            "/api/worker/workflow-tasks/poll",
12424            "/api/worker/activity-tasks/poll",
12425        ] {
12426            assert_eq!(
12427                server.worker_protocol_for(path).as_deref(),
12428                Some(WORKER_PROTOCOL_VERSION),
12429                "unexpected protocol for {path}"
12430            );
12431        }
12432
12433        assert_eq!(
12434            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12435            1
12436        );
12437        assert_eq!(
12438            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12439            1
12440        );
12441        assert!(
12442            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12443                .as_str()
12444                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12445        );
12446        assert!(
12447            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12448                .as_str()
12449                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12450        );
12451    }
12452
12453    #[tokio::test]
12454    async fn query_task_endpoints_send_the_query_feature_protocol() {
12455        let server = MockWorkerServer::start();
12456        let client = Client::builder(server.base_url())
12457            .timeout(Duration::from_secs(2))
12458            .build()
12459            .expect("client");
12460
12461        client
12462            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12463            .await
12464            .expect("query poll");
12465        client
12466            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12467            .await
12468            .expect("query complete");
12469        client
12470            .fail_query_task(
12471                "query-capture",
12472                "capture-worker",
12473                1,
12474                "failed",
12475                "query_rejected",
12476                "QueryFailed",
12477            )
12478            .await
12479            .expect("query fail");
12480
12481        for path in [
12482            "/api/worker/query-tasks/poll",
12483            "/api/worker/query-tasks/query-capture/complete",
12484            "/api/worker/query-tasks/query-capture/fail",
12485        ] {
12486            assert_eq!(
12487                server.worker_protocol_for(path).as_deref(),
12488                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12489                "unexpected protocol for {path}"
12490            );
12491        }
12492
12493        assert_eq!(
12494            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12495            1
12496        );
12497        assert!(
12498            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12499                .as_str()
12500                .is_some_and(|id| id.starts_with("rust-query-poll-"))
12501        );
12502    }
12503
12504    #[tokio::test]
12505    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12506        let server = MockWorkerServer::transient_worker_failures();
12507        let client = Client::builder(server.base_url())
12508            .timeout(Duration::from_secs(2))
12509            .build()
12510            .expect("client");
12511
12512        client
12513            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12514            .await
12515            .expect("workflow poll retry");
12516        client
12517            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12518            .await
12519            .expect("activity poll retry");
12520        client
12521            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12522            .await
12523            .expect("query poll retry");
12524
12525        for path in [
12526            "/api/worker/workflow-tasks/poll",
12527            "/api/worker/activity-tasks/poll",
12528            "/api/worker/query-tasks/poll",
12529        ] {
12530            let bodies = server.request_bodies(path);
12531            assert_eq!(bodies.len(), 2, "{path} must be retried once");
12532            assert_eq!(
12533                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12534                "{path} must preserve the request binding across retry"
12535            );
12536        }
12537    }
12538
12539    #[tokio::test]
12540    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12541        let server = MockWorkerServer::consecutive_poll_failures(2);
12542        let client = Client::builder(server.base_url())
12543            .timeout(Duration::from_secs(2))
12544            .build()
12545            .expect("client");
12546        let mut worker = Worker::new(client, "capture")
12547            .worker_id("capture-worker")
12548            .poll_timeout(Duration::from_millis(10))
12549            .retry_policy(WorkerRetryPolicy {
12550                max_retries: 2,
12551                initial_backoff: Duration::from_millis(1),
12552                max_backoff: Duration::from_millis(1),
12553            });
12554        worker.register_workflow(
12555            "capture.workflow",
12556            |_ctx, _input| async move { Ok(Value::Null) },
12557        );
12558        worker.register_activity(
12559            "capture.activity",
12560            |_ctx, _input| async move { Ok(Value::Null) },
12561        );
12562        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12563            Ok(Value::Null)
12564        });
12565
12566        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12567
12568        for path in [
12569            "/api/worker/workflow-tasks/poll",
12570            "/api/worker/activity-tasks/poll",
12571            "/api/worker/query-tasks/poll",
12572        ] {
12573            let bodies = server.request_bodies(path);
12574            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12575            assert!(
12576                bodies
12577                    .iter()
12578                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12579                "{path} must preserve one request binding across every retry"
12580            );
12581        }
12582    }
12583
12584    #[tokio::test]
12585    async fn query_protocol_rejection_from_older_server_is_typed() {
12586        let server = MockWorkerServer::reject_query_protocol();
12587        let client = Client::builder(server.base_url())
12588            .timeout(Duration::from_secs(2))
12589            .build()
12590            .expect("client");
12591
12592        let error = client
12593            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12594            .await
12595            .expect_err("server below query protocol floor must reject");
12596        let Error::Protocol(failure) = error else {
12597            panic!("expected typed protocol failure");
12598        };
12599
12600        assert_eq!(failure.status, 400);
12601        assert_eq!(failure.reason, "unsupported_protocol_version");
12602        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12603        assert_eq!(
12604            failure.requested_version.as_deref(),
12605            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12606        );
12607        assert_eq!(
12608            server
12609                .worker_protocol_for("/api/worker/query-tasks/poll")
12610                .as_deref(),
12611            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12612        );
12613    }
12614
12615    #[tokio::test]
12616    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12617        let server = MockWorkerServer::reject_query_protocol();
12618        let client = Client::builder(server.base_url())
12619            .timeout(Duration::from_secs(2))
12620            .build()
12621            .expect("client");
12622        let mut worker = Worker::new(client, "rust-workers")
12623            .worker_id("baseline-worker")
12624            .poll_timeout(Duration::from_millis(10));
12625
12626        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12627            Ok(Value::Null)
12628        });
12629
12630        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12631        assert_eq!(
12632            server
12633                .worker_protocol_for("/api/worker/workflow-tasks/poll")
12634                .as_deref(),
12635            Some(WORKER_PROTOCOL_VERSION)
12636        );
12637        assert_eq!(
12638            server.worker_protocol_for("/api/worker/query-tasks/poll"),
12639            None,
12640            "a worker without query handlers must not use the query-task endpoint"
12641        );
12642    }
12643
12644    #[tokio::test]
12645    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12646        let server = MockWorkerServer::reject_query_completion();
12647        let client = Client::builder(server.base_url())
12648            .timeout(Duration::from_secs(2))
12649            .build()
12650            .expect("client");
12651
12652        let error = client
12653            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12654            .await
12655            .expect_err("expired completion must be rejected");
12656        let Error::QueryFailed(failure) = error else {
12657            panic!("expected typed query failure");
12658        };
12659        assert_eq!(failure.status, 409);
12660        assert_eq!(failure.reason, "query_task_timed_out");
12661
12662        let mut worker = Worker::new(client, "rust-workers")
12663            .worker_id("late-worker")
12664            .poll_timeout(Duration::from_millis(10));
12665        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12666        worker.register_query(
12667            "counter",
12668            "current",
12669            |_ctx, _args| async move { Ok(json!(8)) },
12670        );
12671
12672        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12673        assert_eq!(
12674            worker
12675                .run_once()
12676                .await
12677                .expect("worker continues after late completion"),
12678            0
12679        );
12680        assert_eq!(
12681            server.request_count("/api/worker/query-tasks/query-late/complete"),
12682            2
12683        );
12684        assert_eq!(
12685            server.request_count("/api/worker/query-tasks/query-late/fail"),
12686            0,
12687            "a server completion rejection must not be reported as an encoding failure"
12688        );
12689    }
12690
12691    #[tokio::test]
12692    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
12693        let server = MockWorkerServer::start();
12694        let client = Client::builder(server.base_url())
12695            .timeout(Duration::from_secs(2))
12696            .build()
12697            .expect("client");
12698        let mut worker = Worker::new(client, "rust-workers")
12699            .worker_id("activity-only-worker")
12700            .poll_timeout(Duration::from_millis(10));
12701
12702        worker.register_activity(
12703            "activity.only",
12704            |_ctx, _args| async move { Ok(Value::Null) },
12705        );
12706
12707        worker.run_until(async {}).await.expect("run worker");
12708    }
12709
12710    #[tokio::test]
12711    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
12712        let server = MockWorkerServer::start();
12713        let client = Client::builder(server.base_url())
12714            .timeout(Duration::from_secs(2))
12715            .build()
12716            .expect("client");
12717        let mut worker = Worker::new(client, "rust-workers")
12718            .worker_id("workflow-only-worker")
12719            .poll_timeout(Duration::from_millis(10));
12720
12721        worker.register_workflow(
12722            "workflow.only",
12723            |_ctx, _input| async move { Ok(Value::Null) },
12724        );
12725
12726        worker.run_until(async {}).await.expect("run worker");
12727    }
12728
12729    #[tokio::test]
12730    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
12731        let server = MockWorkerServer::start();
12732        let client = Client::builder(server.base_url())
12733            .timeout(Duration::from_secs(2))
12734            .build()
12735            .expect("client");
12736        let observations = Arc::new(Mutex::new(Vec::new()));
12737        let observed = Arc::clone(&observations);
12738        let mut worker = Worker::new(client, "rust-workers")
12739            .worker_id("observed-heartbeat-worker")
12740            .poll_timeout(Duration::from_millis(10))
12741            .on_worker_heartbeat(move |observation| {
12742                observed
12743                    .lock()
12744                    .expect("heartbeat observations")
12745                    .push(observation.clone());
12746            });
12747
12748        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
12749            Ok(Value::Null)
12750        });
12751        let acknowledged = Arc::clone(&observations);
12752        worker
12753            .run_until(async move {
12754                tokio::time::timeout(Duration::from_secs(2), async move {
12755                    loop {
12756                        if !acknowledged
12757                            .lock()
12758                            .expect("heartbeat observations")
12759                            .is_empty()
12760                        {
12761                            break;
12762                        }
12763                        tokio::time::sleep(Duration::from_millis(1)).await;
12764                    }
12765                })
12766                .await
12767                .expect("heartbeat acknowledgement within timeout");
12768            })
12769            .await
12770            .expect("run worker");
12771
12772        let observations = observations.lock().expect("heartbeat observations");
12773        let first = observations.first().expect("heartbeat acknowledgement");
12774        assert_eq!(first.worker_id, "observed-heartbeat-worker");
12775        assert_eq!(first.task_queue, "rust-workers");
12776        assert!(first.acknowledged_at_unix_millis > 0);
12777        assert_eq!(first.acknowledgement, json!({}));
12778    }
12779
12780    #[tokio::test]
12781    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
12782        let server = MockWorkerServer::delayed_heartbeat_worker();
12783        let client = Client::builder(server.base_url())
12784            .timeout(Duration::from_secs(3))
12785            .build()
12786            .expect("client");
12787        let observations = Arc::new(Mutex::new(Vec::new()));
12788        let observed = Arc::clone(&observations);
12789        let mut worker = Worker::new(client, "rust-snapshot-workers")
12790            .worker_id("rust-snapshot-worker")
12791            .poll_timeout(Duration::from_millis(10))
12792            .on_worker_heartbeat(move |observation| {
12793                observed
12794                    .lock()
12795                    .expect("heartbeat observations")
12796                    .push(observation.clone());
12797            });
12798
12799        worker.register_workflow("snapshot", |ctx, _input| async move {
12800            ctx.wait_signal("finish").await?;
12801            Ok(json!({"status": "finished"}))
12802        });
12803        worker.register_query("snapshot", "current", |ctx, _args| async move {
12804            Ok(json!(ctx
12805                .signals("increment")
12806                .iter()
12807                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12808                .sum::<i64>()))
12809        });
12810        worker.register_activity("cancel-aware", |_ctx, _args| async move {
12811            Ok(json!({"late": "completion"}))
12812        });
12813
12814        worker
12815            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
12816            .await
12817            .expect("delayed heartbeat must allow a clean worker shutdown");
12818
12819        let observations = observations.lock().expect("heartbeat observations");
12820        assert!(
12821            observations.len() >= 3,
12822            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
12823        );
12824        assert!(
12825            observations.windows(2).all(|pair| {
12826                pair[1].acknowledged_at_unix_millis
12827                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
12828                    >= 850
12829            }),
12830            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
12831        );
12832        drop(observations);
12833
12834        let heartbeat_times = server.request_times("/api/worker/heartbeat");
12835        let delayed_request_at = *heartbeat_times
12836            .get(1)
12837            .expect("intentionally delayed heartbeat request");
12838        let delay_window_start = delayed_request_at + Duration::from_millis(100);
12839        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
12840        for path in [
12841            "/api/worker/workflow-tasks/poll",
12842            "/api/worker/activity-tasks/poll",
12843            "/api/worker/query-tasks/poll",
12844        ] {
12845            assert!(
12846                server
12847                    .request_times(path)
12848                    .iter()
12849                    .any(|received_at| *received_at >= delay_window_start
12850                        && *received_at <= delay_window_end),
12851                "{path} must keep polling while a heartbeat acknowledgement is delayed"
12852            );
12853        }
12854        assert!(
12855            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
12856            "workflow work must be settled"
12857        );
12858        assert!(
12859            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
12860            "activity work must be settled"
12861        );
12862        assert!(
12863            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
12864            "query work must be settled"
12865        );
12866    }
12867
12868    #[tokio::test]
12869    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
12870        let server = MockWorkerServer::heartbeat_retry_worker();
12871        let client = Client::builder(server.base_url())
12872            .timeout(Duration::from_secs(2))
12873            .build()
12874            .expect("client");
12875        let observations = Arc::new(Mutex::new(Vec::new()));
12876        let observed = Arc::clone(&observations);
12877        let worker = Worker::new(client, "rust-workers")
12878            .worker_id("heartbeat-retry-worker")
12879            .retry_policy(WorkerRetryPolicy {
12880                max_retries: 1,
12881                initial_backoff: Duration::from_millis(300),
12882                max_backoff: Duration::from_millis(300),
12883            })
12884            .on_worker_heartbeat(move |observation| {
12885                observed
12886                    .lock()
12887                    .expect("heartbeat observations")
12888                    .push(observation.clone());
12889            });
12890
12891        worker
12892            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
12893            .await
12894            .expect("retryable heartbeat failure must remain bounded and recover");
12895
12896        let observations = observations.lock().expect("heartbeat observations");
12897        assert!(observations.len() >= 3, "heartbeat retry must recover");
12898        assert!(
12899            observations.windows(2).all(|pair| {
12900                pair[1]
12901                    .acknowledged_at_unix_millis
12902                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
12903                    >= 850
12904            }),
12905            "a successful retry must start a fresh advertised cadence: {observations:?}"
12906        );
12907        assert_eq!(
12908            server.request_count("/api/worker/heartbeat"),
12909            observations.len() + 1,
12910            "one retryable failure must add exactly one bounded request"
12911        );
12912    }
12913
12914    #[tokio::test]
12915    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
12916        let server = MockWorkerServer::waiting_query_worker();
12917        let client = Client::builder(server.base_url())
12918            .timeout(Duration::from_secs(2))
12919            .build()
12920            .expect("client");
12921        let observations = Arc::new(Mutex::new(Vec::new()));
12922        let observed = Arc::clone(&observations);
12923        let mut worker = Worker::new(client, "rust-snapshot-workers")
12924            .worker_id("rust-snapshot-worker")
12925            .poll_timeout(Duration::from_millis(10))
12926            .on_worker_heartbeat(move |observation| {
12927                observed
12928                    .lock()
12929                    .expect("heartbeat observations")
12930                    .push(observation.clone());
12931            });
12932
12933        worker.register_workflow("snapshot", |ctx, _input| async move {
12934            ctx.wait_signal("finish").await?;
12935            Ok(json!({"status": "finished"}))
12936        });
12937        worker.register_query("snapshot", "current", |ctx, _args| async move {
12938            let current = ctx
12939                .signals("increment")
12940                .iter()
12941                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12942                .sum::<i64>();
12943            Ok(json!(current))
12944        });
12945        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
12946
12947        worker
12948            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
12949            .await
12950            .expect("pending workflow and query poller must remain live until shutdown");
12951
12952        assert!(
12953            observations.lock().expect("heartbeat observations").len() >= 4,
12954            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
12955        );
12956        assert!(
12957            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
12958            "workflow polling must continue after empty replay acknowledgements"
12959        );
12960        assert!(
12961            server.request_count("/api/worker/query-tasks/poll") >= 2,
12962            "query polling must continue after serving the current query"
12963        );
12964        assert_eq!(
12965            server.request_body("/api/worker/register")["capabilities"],
12966            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
12967        );
12968        assert_eq!(
12969            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
12970            json!({
12971                "queries": ["current"],
12972                "updates": ["replace"],
12973            })
12974        );
12975
12976        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
12977        assert_eq!(
12978            opened["commands"],
12979            json!([{
12980                "type": "open_signal_wait",
12981                "signal_name": "finish",
12982            }])
12983        );
12984
12985        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
12986            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
12987            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
12988            let failure = server.request_body(&fail_path);
12989            assert_eq!(
12990                failure["failure"]["type"],
12991                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
12992            );
12993            assert_eq!(server.request_count(&completion_path), 0);
12994        }
12995
12996        let query_completion =
12997            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
12998        assert_eq!(query_completion["result"], json!(8));
12999
13000        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13001        assert_eq!(
13002            server.request_count(terminal_path),
13003            1,
13004            "the matching signal must settle the workflow exactly once"
13005        );
13006        let terminal = server.request_body(terminal_path);
13007        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13008        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13009        assert_eq!(
13010            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13011                .expect("terminal workflow result"),
13012            json!({"status": "finished"})
13013        );
13014    }
13015
13016    #[tokio::test]
13017    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13018        let server = MockWorkerServer::transient_worker_failures();
13019        let client = Client::builder(server.base_url())
13020            .timeout(Duration::from_secs(2))
13021            .build()
13022            .expect("client");
13023        let mut worker = Worker::new(client, "rust-workers")
13024            .worker_id("retry-worker")
13025            .poll_timeout(Duration::from_millis(10))
13026            .retry_policy(WorkerRetryPolicy {
13027                max_retries: 2,
13028                initial_backoff: Duration::from_millis(1),
13029                max_backoff: Duration::from_millis(1),
13030            });
13031        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13032        worker.register_activity(
13033            "counter.activity",
13034            |_ctx, _input| async move { Ok(Value::Null) },
13035        );
13036        worker.register_query(
13037            "counter",
13038            "current",
13039            |_ctx, _args| async move { Ok(json!(8)) },
13040        );
13041
13042        worker
13043            .run_until(tokio::time::sleep(Duration::from_millis(75)))
13044            .await
13045            .expect("transient failures must not stop the worker");
13046
13047        for path in [
13048            "/api/worker/heartbeat",
13049            "/api/worker/workflow-tasks/poll",
13050            "/api/worker/activity-tasks/poll",
13051            "/api/worker/query-tasks/poll",
13052        ] {
13053            assert!(
13054                server.request_count(path) >= 2,
13055                "{path} must continue after its transient failure"
13056            );
13057        }
13058    }
13059
13060    #[tokio::test]
13061    async fn worker_bounds_transport_retries() {
13062        let server = MockWorkerServer::unavailable_polls();
13063        let client = Client::builder(server.base_url())
13064            .timeout(Duration::from_secs(2))
13065            .build()
13066            .expect("client");
13067        let mut worker = Worker::new(client, "rust-workers")
13068            .worker_id("bounded-retry-worker")
13069            .poll_timeout(Duration::from_millis(10))
13070            .retry_policy(WorkerRetryPolicy {
13071                max_retries: 2,
13072                initial_backoff: Duration::from_millis(1),
13073                max_backoff: Duration::from_millis(1),
13074            });
13075        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13076
13077        let error = worker.run().await.expect_err("retry bound must terminate");
13078        assert!(matches!(error, Error::Transport(_)));
13079        assert_eq!(
13080            server.request_count("/api/worker/workflow-tasks/poll"),
13081            3,
13082            "one initial request plus exactly two retries"
13083        );
13084    }
13085
13086    #[tokio::test]
13087    async fn worker_retry_policy_can_disable_poll_retries() {
13088        let server = MockWorkerServer::unavailable_polls();
13089        let client = Client::builder(server.base_url())
13090            .timeout(Duration::from_secs(2))
13091            .build()
13092            .expect("client");
13093        let mut worker = Worker::new(client, "rust-workers")
13094            .worker_id("no-retry-worker")
13095            .poll_timeout(Duration::from_millis(10))
13096            .retry_policy(WorkerRetryPolicy {
13097                max_retries: 0,
13098                initial_backoff: Duration::from_millis(1),
13099                max_backoff: Duration::from_millis(1),
13100            });
13101        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13102
13103        let error = worker
13104            .run_once()
13105            .await
13106            .expect_err("disabled retries must return the first transport failure");
13107        assert!(matches!(error, Error::Transport(_)));
13108        assert_eq!(
13109            server.request_count("/api/worker/workflow-tasks/poll"),
13110            1,
13111            "max_retries=0 must send only the initial request"
13112        );
13113    }
13114
13115    #[tokio::test]
13116    async fn worker_does_not_retry_authentication_failures() {
13117        let server = MockWorkerServer::unauthorized_polls();
13118        let client = Client::builder(server.base_url())
13119            .timeout(Duration::from_secs(2))
13120            .build()
13121            .expect("client");
13122        let mut worker = Worker::new(client, "rust-workers")
13123            .worker_id("unauthorized-worker")
13124            .poll_timeout(Duration::from_millis(10));
13125        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13126
13127        let error = worker
13128            .run()
13129            .await
13130            .expect_err("authentication must terminate");
13131        let Error::Http { status, body } = error else {
13132            panic!("expected stable HTTP authentication error");
13133        };
13134        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13135        assert!(body.contains("authentication_failed"));
13136        assert_eq!(
13137            server.request_count("/api/worker/workflow-tasks/poll"),
13138            1,
13139            "authentication failures must not be retried"
13140        );
13141    }
13142
13143    #[derive(Clone, Debug)]
13144    struct CapturedRequest {
13145        path: String,
13146        worker_protocol: Option<String>,
13147        body: String,
13148        received_at: Instant,
13149    }
13150
13151    struct MockWorkerServer {
13152        addr: SocketAddr,
13153        stop: Arc<AtomicBool>,
13154        requests: Arc<Mutex<Vec<CapturedRequest>>>,
13155        thread: Option<thread::JoinHandle<()>>,
13156    }
13157
13158    #[derive(Clone, Copy, Default)]
13159    struct MockWorkerBehavior {
13160        reject_query_protocol: bool,
13161        reject_query_completion: bool,
13162        waiting_query_worker: bool,
13163        complete_named_signal: bool,
13164        poll_failures_per_path: usize,
13165        heartbeat_failures: usize,
13166        heartbeat_failure_request: Option<usize>,
13167        delayed_heartbeat_request: Option<usize>,
13168        heartbeat_response_delay: Duration,
13169        concurrent_requests: bool,
13170        unauthorized_polls: bool,
13171        cancelled_activity: bool,
13172        draining_polls: bool,
13173        workflow_completion_status: Option<&'static str>,
13174        workflow_completion_body: Option<&'static str>,
13175    }
13176
13177    impl MockWorkerServer {
13178        fn start() -> Self {
13179            Self::start_with_behavior(MockWorkerBehavior::default())
13180        }
13181
13182        fn reject_query_protocol() -> Self {
13183            Self::start_with_behavior(MockWorkerBehavior {
13184                reject_query_protocol: true,
13185                ..MockWorkerBehavior::default()
13186            })
13187        }
13188
13189        fn reject_query_completion() -> Self {
13190            Self::start_with_behavior(MockWorkerBehavior {
13191                reject_query_completion: true,
13192                ..MockWorkerBehavior::default()
13193            })
13194        }
13195
13196        fn waiting_query_worker() -> Self {
13197            Self::start_with_behavior(MockWorkerBehavior {
13198                waiting_query_worker: true,
13199                complete_named_signal: true,
13200                ..MockWorkerBehavior::default()
13201            })
13202        }
13203
13204        fn transient_worker_failures() -> Self {
13205            Self::start_with_behavior(MockWorkerBehavior {
13206                poll_failures_per_path: 1,
13207                heartbeat_failures: 1,
13208                ..MockWorkerBehavior::default()
13209            })
13210        }
13211
13212        fn consecutive_poll_failures(count: usize) -> Self {
13213            Self::start_with_behavior(MockWorkerBehavior {
13214                poll_failures_per_path: count,
13215                ..MockWorkerBehavior::default()
13216            })
13217        }
13218
13219        fn delayed_heartbeat_worker() -> Self {
13220            Self::start_with_behavior(MockWorkerBehavior {
13221                waiting_query_worker: true,
13222                delayed_heartbeat_request: Some(2),
13223                heartbeat_response_delay: Duration::from_millis(1_500),
13224                concurrent_requests: true,
13225                cancelled_activity: true,
13226                ..MockWorkerBehavior::default()
13227            })
13228        }
13229
13230        fn heartbeat_retry_worker() -> Self {
13231            Self::start_with_behavior(MockWorkerBehavior {
13232                waiting_query_worker: true,
13233                heartbeat_failure_request: Some(2),
13234                concurrent_requests: true,
13235                ..MockWorkerBehavior::default()
13236            })
13237        }
13238
13239        fn unavailable_polls() -> Self {
13240            Self::start_with_behavior(MockWorkerBehavior {
13241                poll_failures_per_path: usize::MAX,
13242                ..MockWorkerBehavior::default()
13243            })
13244        }
13245
13246        fn unauthorized_polls() -> Self {
13247            Self::start_with_behavior(MockWorkerBehavior {
13248                unauthorized_polls: true,
13249                ..MockWorkerBehavior::default()
13250            })
13251        }
13252
13253        fn cancelled_activity() -> Self {
13254            Self::start_with_behavior(MockWorkerBehavior {
13255                cancelled_activity: true,
13256                ..MockWorkerBehavior::default()
13257            })
13258        }
13259
13260        fn draining_polls() -> Self {
13261            Self::start_with_behavior(MockWorkerBehavior {
13262                draining_polls: true,
13263                ..MockWorkerBehavior::default()
13264            })
13265        }
13266
13267        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13268            Self::start_with_behavior(MockWorkerBehavior {
13269                workflow_completion_status: Some(status),
13270                workflow_completion_body: Some(body),
13271                ..MockWorkerBehavior::default()
13272            })
13273        }
13274
13275        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13276            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13277            listener
13278                .set_nonblocking(true)
13279                .expect("configure mock listener");
13280            let addr = listener.local_addr().expect("mock server address");
13281            let stop = Arc::new(AtomicBool::new(false));
13282            let server_stop = Arc::clone(&stop);
13283            let requests = Arc::new(Mutex::new(Vec::new()));
13284            let server_requests = Arc::clone(&requests);
13285            let thread = thread::spawn(move || {
13286                let mut request_threads = Vec::new();
13287                while !server_stop.load(Ordering::SeqCst) {
13288                    match listener.accept() {
13289                        Ok((mut stream, _)) => {
13290                            if behavior.concurrent_requests {
13291                                let requests = Arc::clone(&server_requests);
13292                                request_threads.push(thread::spawn(move || {
13293                                    handle_mock_worker_request(&mut stream, &requests, behavior)
13294                                }));
13295                            } else {
13296                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
13297                            }
13298                        }
13299                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13300                            let mut index = 0;
13301                            while index < request_threads.len() {
13302                                if request_threads[index].is_finished() {
13303                                    request_threads
13304                                        .swap_remove(index)
13305                                        .join()
13306                                        .expect("join mock request");
13307                                } else {
13308                                    index += 1;
13309                                }
13310                            }
13311                            thread::sleep(Duration::from_millis(5));
13312                        }
13313                        Err(_) => break,
13314                    }
13315                }
13316                for request_thread in request_threads {
13317                    request_thread.join().expect("join mock request");
13318                }
13319            });
13320
13321            Self {
13322                addr,
13323                stop,
13324                requests,
13325                thread: Some(thread),
13326            }
13327        }
13328
13329        fn base_url(&self) -> String {
13330            format!("http://{}", self.addr)
13331        }
13332
13333        fn worker_protocol_for(&self, path: &str) -> Option<String> {
13334            self.requests
13335                .lock()
13336                .expect("captured requests")
13337                .iter()
13338                .find(|request| request.path == path)
13339                .and_then(|request| request.worker_protocol.clone())
13340        }
13341
13342        fn request_count(&self, path: &str) -> usize {
13343            self.requests
13344                .lock()
13345                .expect("captured requests")
13346                .iter()
13347                .filter(|request| request.path == path)
13348                .count()
13349        }
13350
13351        fn request_times(&self, path: &str) -> Vec<Instant> {
13352            self.requests
13353                .lock()
13354                .expect("captured requests")
13355                .iter()
13356                .filter(|request| request.path == path)
13357                .map(|request| request.received_at)
13358                .collect()
13359        }
13360
13361        fn request_body(&self, path: &str) -> Value {
13362            let requests = self.requests.lock().expect("captured requests");
13363            let body = &requests
13364                .iter()
13365                .find(|request| request.path == path)
13366                .unwrap_or_else(|| panic!("missing request for {path}"))
13367                .body;
13368            serde_json::from_str(body).unwrap_or_else(|error| {
13369                panic!("invalid JSON request body for {path}: {error}: {body:?}")
13370            })
13371        }
13372
13373        fn request_bodies(&self, path: &str) -> Vec<Value> {
13374            self.requests
13375                .lock()
13376                .expect("captured requests")
13377                .iter()
13378                .filter(|request| request.path == path)
13379                .map(|request| {
13380                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
13381                        panic!(
13382                            "invalid JSON request body for {path}: {error}: {:?}",
13383                            request.body
13384                        )
13385                    })
13386                })
13387                .collect()
13388        }
13389    }
13390
13391    impl Drop for MockWorkerServer {
13392        fn drop(&mut self) {
13393            self.stop.store(true, Ordering::SeqCst);
13394            let _ = TcpStream::connect(self.addr);
13395
13396            if let Some(thread) = self.thread.take() {
13397                thread.join().expect("join mock server");
13398            }
13399        }
13400    }
13401
13402    fn handle_mock_worker_request(
13403        stream: &mut TcpStream,
13404        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13405        behavior: MockWorkerBehavior,
13406    ) {
13407        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13408        let mut buffer = [0_u8; 8192];
13409        let mut request = Vec::new();
13410
13411        loop {
13412            match stream.read(&mut buffer) {
13413                Ok(0) => break,
13414                Ok(read) => {
13415                    request.extend_from_slice(&buffer[..read]);
13416                    if mock_request_is_complete(&request) {
13417                        break;
13418                    }
13419                }
13420                Err(error)
13421                    if matches!(
13422                        error.kind(),
13423                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13424                    ) =>
13425                {
13426                    break;
13427                }
13428                Err(_) => return,
13429            }
13430        }
13431
13432        let request = String::from_utf8_lossy(&request);
13433        let body = request
13434            .split_once("\r\n\r\n")
13435            .map(|(_, body)| body)
13436            .unwrap_or_default();
13437        let path = request
13438            .lines()
13439            .next()
13440            .and_then(|line| line.split_whitespace().nth(1))
13441            .unwrap_or_default();
13442        let worker_protocol = request.lines().find_map(|line| {
13443            let (name, value) = line.split_once(':')?;
13444            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13445                .then(|| value.trim().to_string())
13446        });
13447        let request_number = {
13448            let mut requests = requests.lock().expect("captured requests");
13449            requests.push(CapturedRequest {
13450                path: path.to_string(),
13451                worker_protocol: worker_protocol.clone(),
13452                body: body.to_string(),
13453                received_at: Instant::now(),
13454            });
13455            requests
13456                .iter()
13457                .filter(|request| request.path == path)
13458                .count()
13459        };
13460
13461        let is_poll = matches!(
13462            path,
13463            "/api/worker/workflow-tasks/poll"
13464                | "/api/worker/activity-tasks/poll"
13465                | "/api/worker/query-tasks/poll"
13466        );
13467        if is_poll && request_number <= behavior.poll_failures_per_path {
13468            return;
13469        }
13470        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
13471            return;
13472        }
13473        if path == "/api/worker/heartbeat"
13474            && behavior.heartbeat_failure_request == Some(request_number)
13475        {
13476            return;
13477        }
13478        if path == "/api/worker/heartbeat"
13479            && behavior.delayed_heartbeat_request == Some(request_number)
13480        {
13481            thread::sleep(behavior.heartbeat_response_delay);
13482        }
13483        if behavior.unauthorized_polls && is_poll {
13484            write_mock_response(
13485                stream,
13486                "401 Unauthorized",
13487                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
13488            );
13489            return;
13490        }
13491        if behavior.draining_polls && is_poll {
13492            write_mock_response(
13493                stream,
13494                "409 Conflict",
13495                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
13496            );
13497            return;
13498        }
13499
13500        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
13501            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
13502            let body = format!(
13503                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
13504            );
13505            write_mock_response(stream, "400 Bad Request", &body);
13506            return;
13507        }
13508
13509        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
13510        {
13511            write_mock_response(
13512                stream,
13513                "409 Conflict",
13514                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
13515            );
13516            return;
13517        }
13518
13519        if behavior.workflow_completion_status.is_some()
13520            && path == "/api/worker/workflow-tasks/poll"
13521            && request_number == 1
13522        {
13523            write_mock_response(
13524                stream,
13525                "200 OK",
13526                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
13527            );
13528            return;
13529        }
13530
13531        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
13532            if let (Some(status), Some(body)) = (
13533                behavior.workflow_completion_status,
13534                behavior.workflow_completion_body,
13535            ) {
13536                write_mock_response(stream, status, body);
13537                return;
13538            }
13539        }
13540
13541        if behavior.waiting_query_worker {
13542            if behavior.complete_named_signal
13543                && path == "/api/worker/workflow-tasks/poll"
13544                && request_number == 1
13545            {
13546                let body = json!({
13547                    "task": {
13548                        "task_id": "snapshot-open",
13549                        "workflow_id": "snapshot-1",
13550                        "run_id": "snapshot-run-1",
13551                        "workflow_type": "snapshot",
13552                        "payload_codec": DEFAULT_CODEC,
13553                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13554                            .expect("Avro workflow arguments"),
13555                        "history_events": [],
13556                        "workflow_task_attempt": 1,
13557                        "lease_owner": "rust-snapshot-worker"
13558                    }
13559                })
13560                .to_string();
13561                write_mock_response(stream, "200 OK", &body);
13562                return;
13563            }
13564
13565            let signal_request = request_number - usize::from(behavior.complete_named_signal);
13566            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
13567            if path == "/api/worker/workflow-tasks/poll"
13568                && signal_request >= 1
13569                && signal_request <= signal_request_limit
13570            {
13571                let finish = behavior.complete_named_signal && signal_request == 3;
13572                let amounts = if signal_request == 1 {
13573                    vec![3]
13574                } else {
13575                    vec![3, 5]
13576                };
13577                let task_id = if signal_request == 1 {
13578                    "snapshot-wait-3"
13579                } else if finish {
13580                    "snapshot-finish"
13581                } else {
13582                    "snapshot-wait-5"
13583                };
13584                let mut history_events = std::iter::once(json!({
13585                    "event_type": "SignalWaitOpened",
13586                    "payload": {"sequence": 1, "signal_name": "finish"}
13587                }))
13588                .chain(amounts.iter().enumerate().map(|(index, amount)| {
13589                    json!({
13590                        "event_type": "SignalReceived",
13591                        "payload": {
13592                            "signal_id": format!("increment-{amount}"),
13593                            "signal_name": "increment",
13594                            "workflow_sequence": index + 2,
13595                            "payload_codec": DEFAULT_CODEC,
13596                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13597                                .expect("Avro signal envelope")
13598                        }
13599                    })
13600                }))
13601                .collect::<Vec<_>>();
13602                let (resume_id, resume_name, resume_arguments) = if finish {
13603                    history_events.push(json!({
13604                        "event_type": "SignalReceived",
13605                        "payload": {
13606                            "signal_id": "finish",
13607                            "signal_name": "finish",
13608                            "workflow_sequence": 4,
13609                            "payload_codec": DEFAULT_CODEC,
13610                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13611                                .expect("Avro finish signal envelope")
13612                        }
13613                    }));
13614                    (
13615                        "finish".to_string(),
13616                        "finish".to_string(),
13617                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
13618                            .expect("Avro finish resume signal"),
13619                    )
13620                } else {
13621                    let amount = amounts.last().expect("amount");
13622                    (
13623                        format!("increment-{amount}"),
13624                        "increment".to_string(),
13625                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13626                            .expect("Avro increment resume signal"),
13627                    )
13628                };
13629                let body = json!({
13630                    "task": {
13631                        "task_id": task_id,
13632                        "workflow_id": "snapshot-1",
13633                        "run_id": "snapshot-run-1",
13634                        "workflow_type": "snapshot",
13635                        "payload_codec": DEFAULT_CODEC,
13636                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13637                            .expect("Avro workflow arguments"),
13638                        "history_events": history_events,
13639                        "workflow_task_attempt": 1,
13640                        "workflow_signal_id": resume_id,
13641                        "signal_name": resume_name,
13642                        "signal_arguments": resume_arguments,
13643                        "lease_owner": "rust-snapshot-worker"
13644                    }
13645                })
13646                .to_string();
13647                write_mock_response(stream, "200 OK", &body);
13648                return;
13649            }
13650
13651            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
13652                let history_events = [3, 5]
13653                    .into_iter()
13654                    .enumerate()
13655                    .map(|(index, amount)| {
13656                        json!({
13657                            "event_type": "SignalReceived",
13658                            "payload": {
13659                                "signal_id": format!("increment-{amount}"),
13660                                "signal_name": "increment",
13661                                "workflow_sequence": index + 2,
13662                                "payload_codec": DEFAULT_CODEC,
13663                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13664                                    .expect("Avro query signal envelope")
13665                            }
13666                        })
13667                    })
13668                    .collect::<Vec<_>>();
13669                let body = json!({
13670                    "task": {
13671                        "query_task_id": "snapshot-current",
13672                        "query_task_attempt": 1,
13673                        "lease_owner": "rust-snapshot-worker",
13674                        "workflow_id": "snapshot-1",
13675                        "run_id": "snapshot-run-1",
13676                        "workflow_type": "snapshot",
13677                        "query_name": "current",
13678                        "payload_codec": DEFAULT_CODEC,
13679                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13680                            .expect("Avro workflow arguments"),
13681                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13682                            .expect("Avro query arguments"),
13683                        "history_events": history_events,
13684                        "run_status": "waiting"
13685                    }
13686                })
13687                .to_string();
13688                write_mock_response(stream, "200 OK", &body);
13689                return;
13690            }
13691
13692            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
13693                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
13694            {
13695                write_mock_response(
13696                    stream,
13697                    "200 OK",
13698                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
13699                );
13700                return;
13701            }
13702
13703            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
13704                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
13705                return;
13706            }
13707
13708            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
13709                write_mock_response(
13710                    stream,
13711                    "200 OK",
13712                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
13713                );
13714                return;
13715            }
13716
13717            if path == "/api/worker/query-tasks/snapshot-current/complete" {
13718                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
13719                return;
13720            }
13721        }
13722
13723        if matches!(
13724            path,
13725            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
13726        ) {
13727            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13728                .expect("typed mock result");
13729            let body = json!({
13730                "result": typed_fidelity_probe().into_json().expect("result projection"),
13731                "result_envelope": result,
13732            })
13733            .to_string();
13734            write_mock_response(stream, "200 OK", &body);
13735            return;
13736        }
13737
13738        if path == "/api/workflows/typed-1" {
13739            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13740                .expect("typed mock result");
13741            let body = json!({
13742                "workflow_id": "typed-1",
13743                "run_id": "run-typed-1",
13744                "workflow_type": "typed.echo",
13745                "status": "completed",
13746                "output": typed_fidelity_probe().into_json().expect("output projection"),
13747                "output_envelope": result,
13748            })
13749            .to_string();
13750            write_mock_response(stream, "200 OK", &body);
13751            return;
13752        }
13753
13754        let (status, body) = match path {
13755            "/api/workflows" => (
13756                "201 Created",
13757                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
13758            ),
13759            "/api/worker/register" if behavior.waiting_query_worker => (
13760                "200 OK",
13761                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
13762            ),
13763            "/api/worker/register" => (
13764                "200 OK",
13765                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
13766            ),
13767            "/api/worker/heartbeat" => ("200 OK", "{}"),
13768            "/api/worker/activity-tasks/poll"
13769                if behavior.cancelled_activity && request_number == 1 =>
13770            {
13771                (
13772                    "200 OK",
13773                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
13774                )
13775            }
13776            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
13777                ("200 OK", r#"{"task":null}"#)
13778            }
13779            "/api/worker/query-tasks/poll"
13780                if behavior.reject_query_completion && request_number == 1 =>
13781            {
13782                (
13783                    "200 OK",
13784                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
13785                )
13786            }
13787            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
13788            "/api/worker/query-tasks/query-capture/complete"
13789            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
13790            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
13791                "200 OK",
13792                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
13793            ),
13794            "/api/worker/activity-tasks/activity-cancel/complete" => (
13795                "409 Conflict",
13796                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
13797            ),
13798            "/api/worker/activity-tasks/activity-typed/complete"
13799            | "/api/worker/activity-tasks/activity-typed/fail"
13800            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
13801            "/api/workflows/counter-1/query/current" => (
13802                "200 OK",
13803                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
13804            ),
13805            "/api/workflows/counter-1/query/missing" => (
13806                "404 Not Found",
13807                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
13808            ),
13809            "/api/workflows/wf-lifecycle/cancel" => (
13810                "200 OK",
13811                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
13812            ),
13813            "/api/workflows/wf-lifecycle/terminate" => (
13814                "200 OK",
13815                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
13816            ),
13817            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
13818                "200 OK",
13819                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
13820            ),
13821            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
13822                "200 OK",
13823                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
13824            ),
13825            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
13826            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
13827                "409 Conflict",
13828                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
13829            ),
13830            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
13831                "200 OK",
13832                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
13833            ),
13834            "/api/workflows/wf-cancelled" => (
13835                "200 OK",
13836                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
13837            ),
13838            "/api/workflows/wf-terminated" => (
13839                "200 OK",
13840                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
13841            ),
13842            "/api/workflows/wf-timed-out" => (
13843                "200 OK",
13844                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
13845            ),
13846            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
13847                "200 OK",
13848                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
13849            ),
13850            "/api/workflows/wf-selected" => (
13851                "200 OK",
13852                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
13853            ),
13854            "/api/workflows/wf-selected/runs/run-selected" => (
13855                "200 OK",
13856                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
13857            ),
13858            _ => ("404 Not Found", r#"{"message":"not found"}"#),
13859        };
13860        write_mock_response(stream, status, body);
13861    }
13862
13863    fn mock_request_is_complete(request: &[u8]) -> bool {
13864        let Some(header_end) = request
13865            .windows(4)
13866            .position(|window| window == b"\r\n\r\n")
13867            .map(|position| position + 4)
13868        else {
13869            return false;
13870        };
13871        let headers = String::from_utf8_lossy(&request[..header_end]);
13872        let content_length = headers.lines().find_map(|line| {
13873            let (name, value) = line.split_once(':')?;
13874            name.eq_ignore_ascii_case("content-length")
13875                .then(|| value.trim().parse::<usize>().ok())
13876                .flatten()
13877        });
13878
13879        request.len() >= header_end + content_length.unwrap_or(0)
13880    }
13881
13882    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
13883        let response = format!(
13884            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
13885            body.len()
13886        );
13887
13888        let _ = stream.write_all(response.as_bytes());
13889        let _ = stream.flush();
13890    }
13891}