Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21    de::DeserializeOwned,
22    ser::{SerializeMap, SerializeSeq},
23    Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34/// Worker-registration capability for server-routed read-only queries.
35pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36/// Worker-registration capability for synchronous workflow updates.
37pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38/// First additive worker protocol that defines query-task transport.
39pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43    "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47    "lease_expired",
48    "query_task_not_found",
49    "query_task_not_leased",
50    "query_task_timed_out",
51];
52
53/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
54pub const AVRO_VALUE_SCHEMA_JSON: &str =
55    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64    ControlPlane,
65    Worker(&'static str),
66}
67
68impl RequestProtocol {
69    fn is_worker(self) -> bool {
70        matches!(self, Self::Worker(_))
71    }
72}
73
74pub type Result<T> = std::result::Result<T, Error>;
75
76#[derive(Debug, Error)]
77pub enum Error {
78    #[error("transport error: {0}")]
79    Transport(#[from] reqwest::Error),
80    #[error("json error: {0}")]
81    Json(#[from] serde_json::Error),
82    #[error("http {status}: {body}")]
83    Http {
84        status: reqwest::StatusCode,
85        body: String,
86    },
87    #[error("codec error: {0}")]
88    Codec(String),
89    #[error(transparent)]
90    QueryFailed(QueryFailure),
91    #[error(transparent)]
92    Protocol(ProtocolFailure),
93    #[error(transparent)]
94    NonDeterministicReplay(ReplayFailure),
95    #[error(transparent)]
96    ChildWorkflowFailed(ChildWorkflowFailure),
97    #[error(transparent)]
98    ActivityFailed(ActivityFailure),
99    #[error(transparent)]
100    WorkflowCommandRejected(WorkflowCommandRejection),
101    #[error(transparent)]
102    WorkflowFailed(WorkflowTerminalOutcome),
103    #[error(transparent)]
104    WorkflowCancelled(WorkflowTerminalOutcome),
105    #[error(transparent)]
106    WorkflowTerminated(WorkflowTerminalOutcome),
107    #[error(transparent)]
108    WorkflowTimedOut(WorkflowTerminalOutcome),
109    #[error(transparent)]
110    ActivityTaskRejected(ActivityTaskRejection),
111    #[error("workflow handler {0:?} is not registered")]
112    WorkflowNotRegistered(String),
113    #[error("activity handler {0:?} is not registered")]
114    ActivityNotRegistered(String),
115    #[error("workflow future yielded without emitting a durable command")]
116    WorkflowYieldedWithoutCommand,
117    #[error("workflow state lock is poisoned")]
118    WorkflowStatePoisoned,
119    #[error("timer duration is too large for the worker protocol")]
120    TimerDurationOverflow,
121    #[error("operation timed out")]
122    Timeout,
123    #[error("worker loop error: {0}")]
124    WorkerLoop(String),
125    #[error("invalid child workflow options: {0}")]
126    InvalidChildWorkflowOptions(String),
127    #[error(transparent)]
128    InvalidActivityOptions(ActivityOptionsError),
129    #[error(transparent)]
130    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
131    #[doc(hidden)]
132    #[error("workflow requested continue as new")]
133    ContinueAsNew(ContinueAsNewRequest),
134}
135
136/// The lifecycle command sent to a workflow execution.
137#[derive(Clone, Copy, Debug, PartialEq, Eq)]
138pub enum WorkflowCommandKind {
139    Cancel,
140    Terminate,
141}
142
143impl WorkflowCommandKind {
144    fn as_str(self) -> &'static str {
145        match self {
146            Self::Cancel => "cancel",
147            Self::Terminate => "terminate",
148        }
149    }
150}
151
152/// Optional structured fields for a cancellation or termination request.
153#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
154pub struct WorkflowCommandOptions {
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub reason: Option<String>,
157    #[serde(skip_serializing_if = "Option::is_none")]
158    pub request_id: Option<String>,
159}
160
161/// Server-enforced timeout policy for a workflow start.
162///
163/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
164/// only bounds how long the caller waits. A server deadline produces a terminal
165/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
166/// `run_timeout`.
167#[derive(Clone, Debug, PartialEq, Eq)]
168pub struct WorkflowStartOptions {
169    pub execution_timeout_seconds: u64,
170    pub run_timeout_seconds: u64,
171}
172
173impl Default for WorkflowStartOptions {
174    fn default() -> Self {
175        Self {
176            execution_timeout_seconds: 3600,
177            run_timeout_seconds: 600,
178        }
179    }
180}
181
182impl WorkflowStartOptions {
183    pub fn new() -> Self {
184        Self::default()
185    }
186
187    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
188        self.execution_timeout_seconds = seconds;
189        self
190    }
191
192    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
193        self.run_timeout_seconds = seconds;
194        self
195    }
196
197    fn validate(&self) -> Result<()> {
198        if self.execution_timeout_seconds == 0 {
199            return Err(Error::Codec(
200                "execution_timeout_seconds must be at least 1".to_string(),
201            ));
202        }
203        if self.run_timeout_seconds == 0 {
204            return Err(Error::Codec(
205                "run_timeout_seconds must be at least 1".to_string(),
206            ));
207        }
208        if self.run_timeout_seconds > self.execution_timeout_seconds {
209            return Err(Error::Codec(
210                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
211            ));
212        }
213
214        Ok(())
215    }
216}
217
218/// Optional routing overrides for a continue-as-new transition.
219///
220/// Omitted values retain the current workflow type and task queue. Server-owned
221/// instance metadata is not accepted here and is carried by the server.
222#[derive(Clone, Debug, Default, PartialEq, Eq)]
223pub struct ContinueAsNewOptions {
224    pub workflow_type: Option<String>,
225    pub task_queue: Option<String>,
226}
227
228impl ContinueAsNewOptions {
229    pub fn new() -> Self {
230        Self::default()
231    }
232
233    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
234        self.workflow_type = Some(workflow_type.into());
235        self
236    }
237
238    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
239        self.task_queue = Some(task_queue.into());
240        self
241    }
242
243    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
244        for (field, value) in [
245            ("workflow_type", self.workflow_type.as_deref()),
246            ("task_queue", self.task_queue.as_deref()),
247        ] {
248            if value.is_some_and(|value| value.trim().is_empty()) {
249                return Err(ContinueAsNewOptionsError {
250                    field,
251                    message: format!("{field} must not be empty"),
252                });
253            }
254        }
255        Ok(())
256    }
257}
258
259/// A stable validation error raised before a continue-as-new command is emitted.
260#[derive(Clone, Debug, Error, PartialEq, Eq)]
261#[error("invalid continue-as-new option {field}: {message}")]
262pub struct ContinueAsNewOptionsError {
263    pub field: &'static str,
264    pub message: String,
265}
266
267/// Public history-budget information attached to the current workflow task.
268#[derive(Clone, Debug, Default, PartialEq, Eq)]
269pub struct WorkflowHistoryBudget {
270    pub event_count: u64,
271    pub size_bytes: Option<u64>,
272    pub continue_as_new_recommended: bool,
273    pub pressure: Option<String>,
274}
275
276#[doc(hidden)]
277#[derive(Clone, Debug)]
278pub struct ContinueAsNewRequest {
279    arguments: AvroValue,
280    options: ContinueAsNewOptions,
281}
282
283impl WorkflowCommandOptions {
284    pub fn new() -> Self {
285        Self::default()
286    }
287
288    pub fn reason(mut self, reason: impl Into<String>) -> Self {
289        self.reason = Some(reason.into());
290        self
291    }
292
293    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
294        self.request_id = Some(request_id.into());
295        self
296    }
297}
298
299/// The accepted, machine-readable result of a lifecycle command.
300#[derive(Clone, Debug, PartialEq)]
301pub struct WorkflowCommandResult {
302    pub command: WorkflowCommandKind,
303    pub workflow_id: String,
304    pub run_id: Option<String>,
305    pub outcome: Option<String>,
306    pub reason: Option<String>,
307    pub command_status: Option<String>,
308    pub raw: Value,
309}
310
311/// A stable rejection returned by instance- or selected-run lifecycle commands.
312#[derive(Clone, Debug, Error)]
313#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
314pub struct WorkflowCommandRejection {
315    pub command: WorkflowCommandKind,
316    pub status: u16,
317    pub reason: String,
318    pub message: String,
319    pub workflow_id: String,
320    pub run_id: Option<String>,
321    pub target_scope: Option<String>,
322    pub body: Value,
323}
324
325/// Stable terminal categories returned by [`WorkflowHandle::result`].
326#[derive(Clone, Copy, Debug, PartialEq, Eq)]
327pub enum WorkflowTerminalKind {
328    Failed,
329    Cancelled,
330    Terminated,
331    TimedOut,
332}
333
334/// A typed terminal workflow outcome with durable identity and failure metadata.
335///
336/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
337/// of parsing its display representation. Fields remain `None` when an older
338/// server did not publish that metadata.
339#[derive(Clone, Debug, Error)]
340#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
341pub struct WorkflowTerminalOutcome {
342    pub kind: WorkflowTerminalKind,
343    pub workflow_id: String,
344    pub run_id: Option<String>,
345    pub reason: String,
346    pub failure_category: Option<String>,
347    pub failure_id: Option<String>,
348    pub exception_type: Option<String>,
349    pub exception_class: Option<String>,
350    pub non_retryable: Option<bool>,
351    pub message: Option<String>,
352    pub exception: Option<Value>,
353    pub raw: Value,
354}
355
356/// A worker-side activity settlement or heartbeat rejected by durable state.
357#[derive(Clone, Debug, Error)]
358#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
359pub struct ActivityTaskRejection {
360    pub operation: String,
361    pub status: u16,
362    pub reason: String,
363    pub task_id: String,
364    pub activity_attempt_id: String,
365    pub cancel_requested: bool,
366    pub can_continue: Option<bool>,
367    pub run_closed_reason: Option<String>,
368    pub body: Value,
369}
370
371/// Stable validation categories for [`ActivityOptions`].
372#[derive(Clone, Copy, Debug, PartialEq, Eq)]
373pub enum ActivityOptionsErrorKind {
374    EmptyTaskQueue,
375    EmptyRetryPolicy,
376    InvalidMaxAttempts,
377    BackoffWithoutRetryBudget,
378    TooManyBackoffIntervals,
379    InvalidBackoffCoefficient,
380    BackoffGenerationTooLarge,
381    BackoffOverflow,
382    EmptyNonRetryableErrorType,
383    TimeoutNotPositive,
384    TimeoutOverflow,
385    TimeoutOrder,
386}
387
388/// A machine-readable activity-options validation failure.
389#[derive(Clone, Debug, Error, PartialEq, Eq)]
390#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
391pub struct ActivityOptionsError {
392    pub kind: ActivityOptionsErrorKind,
393    pub field: Option<&'static str>,
394    pub message: String,
395}
396
397impl ActivityOptionsError {
398    fn new(
399        kind: ActivityOptionsErrorKind,
400        field: Option<&'static str>,
401        message: impl Into<String>,
402    ) -> Self {
403        Self {
404            kind,
405            field,
406            message: message.into(),
407        }
408    }
409}
410
411/// Stable terminal categories returned when an awaited activity does not succeed.
412#[derive(Clone, Copy, Debug, PartialEq, Eq)]
413pub enum ActivityFailureKind {
414    Failed,
415    Cancelled,
416    TimedOut,
417}
418
419/// A stable, machine-readable terminal activity failure.
420///
421/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
422/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
423#[derive(Clone, Debug, Error)]
424#[error("activity failed ({reason}): {message}")]
425pub struct ActivityFailure {
426    pub kind: ActivityFailureKind,
427    pub reason: String,
428    pub message: String,
429    pub activity_execution_id: Option<String>,
430    pub activity_attempt_id: Option<String>,
431    pub activity_type: Option<String>,
432    pub activity_class: Option<String>,
433    pub attempt_number: Option<u64>,
434    pub failure_id: Option<String>,
435    pub failure_category: Option<String>,
436    pub timeout_kind: Option<String>,
437    pub non_retryable: bool,
438    pub exception_type: Option<String>,
439    pub exception_class: Option<String>,
440    pub code: Option<Value>,
441    pub exception: Option<Value>,
442}
443
444/// Stable terminal categories returned when an awaited child does not succeed.
445#[derive(Clone, Copy, Debug, PartialEq, Eq)]
446pub enum ChildWorkflowFailureKind {
447    Failed,
448    Cancelled,
449    Terminated,
450}
451
452/// A stable, machine-readable child workflow failure delivered to its parent.
453///
454/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
455/// of parsing the display message. Child and parent identifiers retain the
456/// relationship recorded in durable history across worker restarts.
457#[derive(Clone, Debug, Error)]
458#[error("child workflow failed ({reason}): {message}")]
459pub struct ChildWorkflowFailure {
460    pub kind: ChildWorkflowFailureKind,
461    pub reason: String,
462    pub message: String,
463    pub parent_workflow_id: Option<String>,
464    pub parent_workflow_run_id: Option<String>,
465    pub child_workflow_id: Option<String>,
466    pub child_workflow_run_id: Option<String>,
467    pub child_workflow_type: Option<String>,
468    pub failure_id: Option<String>,
469    pub failure_category: Option<String>,
470    pub exception_type: Option<String>,
471    pub exception_class: Option<String>,
472    pub non_retryable: bool,
473    pub code: Option<Value>,
474    pub exception: Option<Value>,
475}
476
477/// The identity of one durable workflow execution.
478#[derive(Clone, Debug, PartialEq, Eq)]
479pub struct WorkflowIdentity {
480    pub workflow_id: Option<String>,
481    pub run_id: Option<String>,
482}
483
484/// A successful child result together with its durable parent-child identity.
485#[derive(Clone, Debug, PartialEq)]
486pub struct ChildWorkflowResult {
487    pub parent: WorkflowIdentity,
488    pub child: WorkflowIdentity,
489    pub child_workflow_type: Option<String>,
490    pub result: Value,
491}
492
493/// Lossless successful child result for fixed Avro Value workflows.
494#[derive(Clone, Debug, PartialEq)]
495pub struct ChildWorkflowAvroResult {
496    pub parent: WorkflowIdentity,
497    pub child: WorkflowIdentity,
498    pub child_workflow_type: Option<String>,
499    pub result: AvroValue,
500}
501
502/// Server behavior when a parent closes while its child is still open.
503#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
504pub enum ParentClosePolicy {
505    #[default]
506    Abandon,
507    RequestCancel,
508    Terminate,
509}
510
511impl ParentClosePolicy {
512    fn as_str(self) -> &'static str {
513        match self {
514            Self::Abandon => "abandon",
515            Self::RequestCancel => "request_cancel",
516            Self::Terminate => "terminate",
517        }
518    }
519}
520
521/// Durable retry policy for one child workflow invocation.
522#[derive(Clone, Debug, Default, PartialEq, Eq)]
523pub struct ChildWorkflowRetryPolicy {
524    pub max_attempts: Option<u32>,
525    pub backoff_seconds: Vec<u64>,
526    pub non_retryable_error_types: Vec<String>,
527}
528
529/// Options recorded with a child-workflow command.
530///
531/// The task queue is mandatory so routing is explicit and replay-stable.
532#[derive(Clone, Debug, PartialEq, Eq)]
533pub struct ChildWorkflowOptions {
534    pub task_queue: String,
535    pub parent_close_policy: ParentClosePolicy,
536    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
537    pub execution_timeout_seconds: Option<u64>,
538    pub run_timeout_seconds: Option<u64>,
539}
540
541impl ChildWorkflowOptions {
542    pub fn new(task_queue: impl Into<String>) -> Self {
543        Self {
544            task_queue: task_queue.into(),
545            parent_close_policy: ParentClosePolicy::Abandon,
546            retry_policy: None,
547            execution_timeout_seconds: None,
548            run_timeout_seconds: None,
549        }
550    }
551
552    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
553        self.parent_close_policy = policy;
554        self
555    }
556
557    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
558        self.retry_policy = Some(policy);
559        self
560    }
561
562    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
563        self.execution_timeout_seconds = Some(seconds);
564        self
565    }
566
567    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
568        self.run_timeout_seconds = Some(seconds);
569        self
570    }
571}
572
573/// Backoff intervals for one durable activity retry policy.
574#[derive(Clone, Debug, PartialEq, Eq)]
575pub enum ActivityBackoff {
576    /// Use these intervals between attempts. The server repeats the final
577    /// interval if the retry budget contains more attempts than entries.
578    Explicit(Vec<Duration>),
579    /// Generate one interval for every retry using integer exponential growth.
580    Exponential {
581        initial_interval: Duration,
582        coefficient: u32,
583        maximum_interval: Option<Duration>,
584    },
585}
586
587/// Durable server-side retry policy for one activity execution.
588#[derive(Clone, Debug, Default, PartialEq, Eq)]
589pub struct ActivityRetryPolicy {
590    pub max_attempts: Option<u32>,
591    pub backoff: Option<ActivityBackoff>,
592    pub non_retryable_error_types: Vec<String>,
593}
594
595impl ActivityRetryPolicy {
596    /// Start a policy with a finite attempt budget, including the first attempt.
597    pub fn new(max_attempts: u32) -> Self {
598        Self {
599            max_attempts: Some(max_attempts),
600            ..Self::default()
601        }
602    }
603
604    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
605        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
606        self
607    }
608
609    pub fn exponential_backoff(
610        mut self,
611        initial_interval: Duration,
612        coefficient: u32,
613        maximum_interval: Option<Duration>,
614    ) -> Self {
615        self.backoff = Some(ActivityBackoff::Exponential {
616            initial_interval,
617            coefficient,
618            maximum_interval,
619        });
620        self
621    }
622
623    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
624        self.non_retryable_error_types.push(error_type.into());
625        self
626    }
627
628    pub fn non_retryable_error_types(
629        mut self,
630        error_types: impl IntoIterator<Item = impl Into<String>>,
631    ) -> Self {
632        self.non_retryable_error_types
633            .extend(error_types.into_iter().map(Into::into));
634        self
635    }
636}
637
638/// Options recorded atomically on one deterministic `schedule_activity` command.
639///
640/// Durations are rounded up to whole seconds when encoded, so the server never
641/// receives a shorter timeout or backoff than the caller requested.
642#[derive(Clone, Debug, Default, PartialEq, Eq)]
643pub struct ActivityOptions {
644    pub task_queue: Option<String>,
645    pub retry_policy: Option<ActivityRetryPolicy>,
646    pub start_to_close_timeout: Option<Duration>,
647    pub schedule_to_start_timeout: Option<Duration>,
648    pub schedule_to_close_timeout: Option<Duration>,
649    pub heartbeat_timeout: Option<Duration>,
650}
651
652impl ActivityOptions {
653    pub fn new() -> Self {
654        Self::default()
655    }
656
657    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
658        self.task_queue = Some(task_queue.into());
659        self
660    }
661
662    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
663        self.retry_policy = Some(policy);
664        self
665    }
666
667    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
668        self.start_to_close_timeout = Some(timeout);
669        self
670    }
671
672    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
673        self.schedule_to_start_timeout = Some(timeout);
674        self
675    }
676
677    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
678        self.schedule_to_close_timeout = Some(timeout);
679        self
680    }
681
682    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
683        self.heartbeat_timeout = Some(timeout);
684        self
685    }
686
687    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
688        if self
689            .task_queue
690            .as_deref()
691            .is_some_and(|queue| queue.trim().is_empty())
692        {
693            return Err(ActivityOptionsError::new(
694                ActivityOptionsErrorKind::EmptyTaskQueue,
695                Some("task_queue"),
696                "task_queue must not be empty",
697            ));
698        }
699
700        for (field, value) in [
701            ("start_to_close_timeout", self.start_to_close_timeout),
702            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
703            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
704            ("heartbeat_timeout", self.heartbeat_timeout),
705        ] {
706            if value.is_some_and(|value| value.is_zero()) {
707                return Err(ActivityOptionsError::new(
708                    ActivityOptionsErrorKind::TimeoutNotPositive,
709                    Some(field),
710                    format!("{field} must be positive"),
711                ));
712            }
713        }
714
715        validate_timeout_order(
716            "heartbeat_timeout",
717            self.heartbeat_timeout,
718            "start_to_close_timeout",
719            self.start_to_close_timeout,
720        )?;
721        validate_timeout_order(
722            "start_to_close_timeout",
723            self.start_to_close_timeout,
724            "schedule_to_close_timeout",
725            self.schedule_to_close_timeout,
726        )?;
727        validate_timeout_order(
728            "schedule_to_start_timeout",
729            self.schedule_to_start_timeout,
730            "schedule_to_close_timeout",
731            self.schedule_to_close_timeout,
732        )?;
733
734        Ok(ValidatedActivityOptions {
735            task_queue: self.task_queue.clone(),
736            retry_policy: self
737                .retry_policy
738                .as_ref()
739                .map(validate_activity_retry_policy)
740                .transpose()?,
741            start_to_close_timeout: timeout_seconds(
742                "start_to_close_timeout",
743                self.start_to_close_timeout,
744            )?,
745            schedule_to_start_timeout: timeout_seconds(
746                "schedule_to_start_timeout",
747                self.schedule_to_start_timeout,
748            )?,
749            schedule_to_close_timeout: timeout_seconds(
750                "schedule_to_close_timeout",
751                self.schedule_to_close_timeout,
752            )?,
753            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
754        })
755    }
756}
757
758#[derive(Clone, Debug)]
759struct ValidatedActivityOptions {
760    task_queue: Option<String>,
761    retry_policy: Option<Value>,
762    start_to_close_timeout: Option<u64>,
763    schedule_to_start_timeout: Option<u64>,
764    schedule_to_close_timeout: Option<u64>,
765    heartbeat_timeout: Option<u64>,
766}
767
768fn validate_timeout_order(
769    smaller_name: &'static str,
770    smaller: Option<Duration>,
771    larger_name: &'static str,
772    larger: Option<Duration>,
773) -> std::result::Result<(), ActivityOptionsError> {
774    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
775        return Err(ActivityOptionsError::new(
776            ActivityOptionsErrorKind::TimeoutOrder,
777            Some(smaller_name),
778            format!("{smaller_name} must be <= {larger_name}"),
779        ));
780    }
781    Ok(())
782}
783
784fn timeout_seconds(
785    field: &'static str,
786    value: Option<Duration>,
787) -> std::result::Result<Option<u64>, ActivityOptionsError> {
788    value
789        .map(|value| {
790            activity_protocol_seconds(value).ok_or_else(|| {
791                ActivityOptionsError::new(
792                    ActivityOptionsErrorKind::TimeoutOverflow,
793                    Some(field),
794                    format!("{field} is too large for the worker protocol"),
795                )
796            })
797        })
798        .transpose()
799}
800
801fn duration_seconds_ceil(value: Duration) -> Option<u64> {
802    value
803        .as_secs()
804        .checked_add(u64::from(value.subsec_nanos() > 0))
805}
806
807fn activity_protocol_seconds(value: Duration) -> Option<u64> {
808    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
809}
810
811fn validate_activity_retry_policy(
812    policy: &ActivityRetryPolicy,
813) -> std::result::Result<Value, ActivityOptionsError> {
814    if policy.max_attempts.is_none()
815        && policy.backoff.is_none()
816        && policy.non_retryable_error_types.is_empty()
817    {
818        return Err(ActivityOptionsError::new(
819            ActivityOptionsErrorKind::EmptyRetryPolicy,
820            Some("retry_policy"),
821            "retry_policy must configure at least one field",
822        ));
823    }
824    if policy.max_attempts == Some(0) {
825        return Err(ActivityOptionsError::new(
826            ActivityOptionsErrorKind::InvalidMaxAttempts,
827            Some("retry_policy.max_attempts"),
828            "max_attempts must be >= 1",
829        ));
830    }
831    if policy
832        .non_retryable_error_types
833        .iter()
834        .any(|error_type| error_type.trim().is_empty())
835    {
836        return Err(ActivityOptionsError::new(
837            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
838            Some("retry_policy.non_retryable_error_types"),
839            "non_retryable_error_types must not contain empty values",
840        ));
841    }
842
843    let backoff_seconds = match &policy.backoff {
844        None => None,
845        Some(backoff) => {
846            let max_attempts = policy.max_attempts.ok_or_else(|| {
847                ActivityOptionsError::new(
848                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
849                    Some("retry_policy.backoff"),
850                    "backoff requires max_attempts",
851                )
852            })?;
853            let retry_count = max_attempts.saturating_sub(1) as usize;
854            let intervals = match backoff {
855                ActivityBackoff::Explicit(intervals) => {
856                    if intervals.len() > retry_count {
857                        return Err(ActivityOptionsError::new(
858                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
859                            Some("retry_policy.backoff"),
860                            "backoff interval count must not exceed max_attempts - 1",
861                        ));
862                    }
863                    intervals.clone()
864                }
865                ActivityBackoff::Exponential {
866                    initial_interval,
867                    coefficient,
868                    maximum_interval,
869                } => {
870                    if *coefficient < 1 {
871                        return Err(ActivityOptionsError::new(
872                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
873                            Some("retry_policy.backoff.coefficient"),
874                            "backoff coefficient must be >= 1",
875                        ));
876                    }
877                    if retry_count > 10_000 {
878                        return Err(ActivityOptionsError::new(
879                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
880                            Some("retry_policy.max_attempts"),
881                            "generated backoff supports at most 10000 retry intervals",
882                        ));
883                    }
884                    let mut current = *initial_interval;
885                    let mut intervals = Vec::with_capacity(retry_count);
886                    for _ in 0..retry_count {
887                        let interval = maximum_interval
888                            .map(|maximum| current.min(maximum))
889                            .unwrap_or(current);
890                        intervals.push(interval);
891                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
892                            break;
893                        }
894                        current = current.checked_mul(*coefficient).ok_or_else(|| {
895                            ActivityOptionsError::new(
896                                ActivityOptionsErrorKind::BackoffOverflow,
897                                Some("retry_policy.backoff"),
898                                "generated backoff interval overflowed",
899                            )
900                        })?;
901                    }
902                    intervals
903                }
904            };
905            Some(
906                intervals
907                    .into_iter()
908                    .map(|interval| {
909                        activity_protocol_seconds(interval).ok_or_else(|| {
910                            ActivityOptionsError::new(
911                                ActivityOptionsErrorKind::BackoffOverflow,
912                                Some("retry_policy.backoff"),
913                                "backoff interval is too large for the worker protocol",
914                            )
915                        })
916                    })
917                    .collect::<std::result::Result<Vec<_>, _>>()?,
918            )
919        }
920    };
921
922    let mut encoded = serde_json::Map::new();
923    if let Some(max_attempts) = policy.max_attempts {
924        encoded.insert("max_attempts".to_string(), json!(max_attempts));
925    }
926    if let Some(backoff_seconds) = backoff_seconds {
927        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
928    }
929    if !policy.non_retryable_error_types.is_empty() {
930        let mut canonical_error_types = Vec::new();
931        for error_type in policy
932            .non_retryable_error_types
933            .iter()
934            .map(|error_type| error_type.trim())
935        {
936            if !canonical_error_types.contains(&error_type) {
937                canonical_error_types.push(error_type);
938            }
939        }
940        encoded.insert(
941            "non_retryable_error_types".to_string(),
942            json!(canonical_error_types),
943        );
944    }
945    Ok(Value::Object(encoded))
946}
947
948/// A stable, machine-readable failure raised when workflow code no longer
949/// reconstructs the durable command stream recorded in history.
950#[derive(Clone, Debug, Error)]
951#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
952pub struct ReplayFailure {
953    pub reason: String,
954    pub sequence: Option<u64>,
955    pub expected: Option<String>,
956    pub actual: Option<String>,
957    pub message: String,
958}
959
960impl ReplayFailure {
961    fn new(
962        reason: impl Into<String>,
963        sequence: Option<u64>,
964        expected: Option<String>,
965        actual: Option<String>,
966        message: impl Into<String>,
967    ) -> Self {
968        Self {
969            reason: reason.into(),
970            sequence,
971            expected,
972            actual,
973            message: message.into(),
974        }
975    }
976}
977
978/// A stable, machine-readable workflow query or query-task settlement failure.
979#[derive(Clone, Debug, Error)]
980#[error("query failed ({reason}, HTTP {status}): {message}")]
981pub struct QueryFailure {
982    pub status: u16,
983    pub reason: String,
984    pub message: String,
985    pub body: Value,
986}
987
988/// A stable failure returned when a server rejects an SDK protocol version.
989#[derive(Clone, Debug, Error)]
990#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
991pub struct ProtocolFailure {
992    pub status: u16,
993    pub reason: String,
994    pub message: String,
995    pub supported_version: Option<String>,
996    pub requested_version: Option<String>,
997    pub body: Value,
998}
999
1000#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1001pub struct PayloadEnvelope {
1002    pub codec: String,
1003    pub blob: String,
1004}
1005
1006impl PayloadEnvelope {
1007    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1008        encode_payload(value, DEFAULT_CODEC)
1009    }
1010
1011    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1012        encode_payload(value, JSON_CODEC)
1013    }
1014
1015    /// Encode an explicit typed value, including the bytes branch that JSON
1016    /// serialization cannot represent.
1017    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1018        encode_avro_value(value)
1019    }
1020}
1021
1022/// Native adapter for the fixed language-neutral Avro Value schema.
1023#[derive(Clone, Debug, PartialEq)]
1024pub enum AvroValue {
1025    Null,
1026    Boolean(bool),
1027    Long(i64),
1028    Double(f64),
1029    Bytes(Vec<u8>),
1030    String(String),
1031    Array(Vec<AvroValue>),
1032    Map(BTreeMap<String, AvroValue>),
1033}
1034
1035impl AvroValue {
1036    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1037        Self::from_serde_value(
1038            serde_value::to_value(value).map_err(|error| {
1039                Error::Codec(format!("could not adapt value for Avro: {error}"))
1040            })?,
1041        )
1042    }
1043
1044    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1045        use serde_value::Value as SerdeValue;
1046
1047        match value {
1048            SerdeValue::Unit => Ok(Self::Null),
1049            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1050            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1051            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1052            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1053            SerdeValue::I64(value) => Ok(Self::Long(value)),
1054            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1055            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1056            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1057            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1058                Error::Codec(
1059                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1060                        .to_string(),
1061                )
1062            }),
1063            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1064            SerdeValue::F64(value) => Self::finite_double(value),
1065            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1066            SerdeValue::String(value) => Ok(Self::String(value)),
1067            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1068            SerdeValue::Option(None) => Ok(Self::Null),
1069            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1070                Self::from_serde_value(*value)
1071            }
1072            SerdeValue::Seq(values) => values
1073                .into_iter()
1074                .map(Self::from_serde_value)
1075                .collect::<Result<Vec<_>>>()
1076                .map(Self::Array),
1077            SerdeValue::Map(values) => values
1078                .into_iter()
1079                .map(|(key, value)| {
1080                    let SerdeValue::String(key) = key else {
1081                        return Err(Error::Codec(
1082                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1083                        ));
1084                    };
1085
1086                    Ok((key, Self::from_serde_value(value)?))
1087                })
1088                .collect::<Result<BTreeMap<_, _>>>()
1089                .map(Self::Map),
1090        }
1091    }
1092
1093    fn finite_double(value: f64) -> Result<Self> {
1094        if !value.is_finite() {
1095            return Err(Error::Codec(
1096                "non_finite_float: Avro Value doubles must be finite".to_string(),
1097            ));
1098        }
1099
1100        Ok(Self::Double(value))
1101    }
1102
1103    fn into_json(self) -> Result<Value> {
1104        match self {
1105            Self::Null => Ok(Value::Null),
1106            Self::Boolean(value) => Ok(Value::Bool(value)),
1107            Self::Long(value) => Ok(Value::Number(value.into())),
1108            Self::Double(value) => serde_json::Number::from_f64(value)
1109                .map(Value::Number)
1110                .ok_or_else(|| {
1111                    Error::Codec(
1112                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1113                    )
1114                }),
1115            Self::Bytes(value) => Ok(json!({
1116                "$type": "bytes",
1117                "base64": BASE64.encode(value),
1118            })),
1119            Self::String(value) => Ok(Value::String(value)),
1120            Self::Array(values) => values
1121                .into_iter()
1122                .map(Self::into_json)
1123                .collect::<Result<Vec<_>>>()
1124                .map(Value::Array),
1125            Self::Map(values) => values
1126                .into_iter()
1127                .map(|(key, value)| Ok((key, value.into_json()?)))
1128                .collect::<Result<serde_json::Map<_, _>>>()
1129                .map(Value::Object),
1130        }
1131    }
1132
1133    fn into_serde_value(self) -> serde_value::Value {
1134        use serde_value::Value as SerdeValue;
1135
1136        match self {
1137            Self::Null => SerdeValue::Unit,
1138            Self::Boolean(value) => SerdeValue::Bool(value),
1139            Self::Long(value) => SerdeValue::I64(value),
1140            Self::Double(value) => SerdeValue::F64(value),
1141            Self::Bytes(value) => SerdeValue::Bytes(value),
1142            Self::String(value) => SerdeValue::String(value),
1143            Self::Array(values) => {
1144                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1145            }
1146            Self::Map(values) => SerdeValue::Map(
1147                values
1148                    .into_iter()
1149                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1150                    .collect(),
1151            ),
1152        }
1153    }
1154
1155    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1156        self.into_serde_value().deserialize_into().map_err(|error| {
1157            Error::Codec(format!(
1158                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1159            ))
1160        })
1161    }
1162}
1163
1164impl Serialize for AvroValue {
1165    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1166    where
1167        S: Serializer,
1168    {
1169        match self {
1170            Self::Null => serializer.serialize_unit(),
1171            Self::Boolean(value) => serializer.serialize_bool(*value),
1172            Self::Long(value) => serializer.serialize_i64(*value),
1173            Self::Double(value) => serializer.serialize_f64(*value),
1174            Self::Bytes(value) => serializer.serialize_bytes(value),
1175            Self::String(value) => serializer.serialize_str(value),
1176            Self::Array(values) => {
1177                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1178                for value in values {
1179                    sequence.serialize_element(value)?;
1180                }
1181                sequence.end()
1182            }
1183            Self::Map(values) => {
1184                let mut map = serializer.serialize_map(Some(values.len()))?;
1185                for (key, value) in values {
1186                    map.serialize_entry(key, value)?;
1187                }
1188                map.end()
1189            }
1190        }
1191    }
1192}
1193
1194pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1195    let datum = avro_value_to_datum(value)?;
1196    let datum = to_avro_datum(avro_value_schema()?, datum)
1197        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1198    let mut bytes = Vec::with_capacity(datum.len() + 10);
1199    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1200    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1201    bytes.extend_from_slice(&datum);
1202    Ok(PayloadEnvelope {
1203        codec: DEFAULT_CODEC.to_string(),
1204        blob: BASE64.encode(bytes),
1205    })
1206}
1207
1208pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1209    if envelope.codec != DEFAULT_CODEC {
1210        return Err(Error::Codec(format!(
1211            "unsupported payload codec {:?}",
1212            envelope.codec
1213        )));
1214    }
1215    decode_avro_value_blob(&envelope.blob)
1216}
1217
1218pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1219    let blob = match codec {
1220        JSON_CODEC => serde_json::to_string(value)?,
1221        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1222        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1223    };
1224
1225    Ok(PayloadEnvelope {
1226        codec: codec.to_string(),
1227        blob,
1228    })
1229}
1230
1231pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1232    match envelope.codec.as_str() {
1233        JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1234        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1235        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1236    }
1237}
1238
1239#[cfg(test)]
1240fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1241    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1242}
1243
1244fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1245    if value.is_null() {
1246        return Ok(Value::Null);
1247    }
1248
1249    if let Some(object) = value.as_object() {
1250        if let (Some(codec), Some(blob)) = (
1251            object.get("codec").and_then(Value::as_str),
1252            object.get("blob").and_then(Value::as_str),
1253        ) {
1254            return decode_blob(blob, codec);
1255        }
1256    }
1257
1258    if let Some(blob) = value.as_str() {
1259        return decode_blob(blob, fallback_codec);
1260    }
1261
1262    Ok(value.clone())
1263}
1264
1265fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1266    let envelope = match codec {
1267        DEFAULT_CODEC => encode_avro_value(value)?,
1268        JSON_CODEC => PayloadEnvelope {
1269            codec: JSON_CODEC.to_string(),
1270            blob: serde_json::to_string(&value.clone().into_json()?)?,
1271        },
1272        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1273    };
1274    Ok(serde_json::to_value(envelope)?)
1275}
1276
1277fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1278    if value.is_null() {
1279        return Ok(AvroValue::Null);
1280    }
1281
1282    if let Some(object) = value.as_object() {
1283        if let (Some(codec), Some(blob)) = (
1284            object.get("codec").and_then(Value::as_str),
1285            object.get("blob").and_then(Value::as_str),
1286        ) {
1287            return match codec {
1288                DEFAULT_CODEC => decode_avro_value_blob(blob),
1289                JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1290                other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1291            };
1292        }
1293    }
1294
1295    if let Some(blob) = value.as_str() {
1296        return match fallback_codec {
1297            DEFAULT_CODEC => decode_avro_value_blob(blob),
1298            JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1299            other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1300        };
1301    }
1302
1303    AvroValue::from_serialize(value)
1304}
1305
1306fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1307    match value {
1308        AvroValue::Null => AvroValue::Array(Vec::new()),
1309        AvroValue::Array(_) => value,
1310        other => AvroValue::Array(vec![other]),
1311    }
1312}
1313
1314fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1315    match codec {
1316        JSON_CODEC => Ok(serde_json::from_str(blob)?),
1317        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1318        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1319    }
1320}
1321
1322fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1323    let bytes = BASE64.decode(blob).map_err(|err| {
1324        Error::Codec(format!(
1325            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1326        ))
1327    })?;
1328
1329    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1330        return Err(Error::Codec(
1331            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1332        ));
1333    }
1334
1335    let fingerprint: [u8; 8] = bytes[2..10]
1336        .try_into()
1337        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1338    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1339        return Err(Error::Codec(format!(
1340            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1341            fingerprint
1342                .iter()
1343                .map(|byte| format!("{byte:02x}"))
1344                .collect::<String>()
1345        )));
1346    }
1347
1348    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1349    // The current fingerprint selects the current immutable schema, so reader
1350    // resolution would only re-walk the same recursive union. Future retained
1351    // writer fingerprints supply a distinct reader schema in this branch.
1352    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1353    if datum_reader.truncated {
1354        return Err(Error::Codec(
1355            "invalid_payload_framing: truncated Avro Value datum".to_string(),
1356        ));
1357    }
1358    let datum = datum.map_err(|err| {
1359        Error::Codec(format!(
1360            "invalid_payload_framing: malformed Avro Value datum: {err}"
1361        ))
1362    })?;
1363    if datum_reader.remaining() != 0 {
1364        return Err(Error::Codec(format!(
1365            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1366            datum_reader.remaining()
1367        )));
1368    }
1369    avro_value_from_datum(datum)
1370}
1371
1372struct StrictAvroDatumReader<'a> {
1373    bytes: &'a [u8],
1374    offset: usize,
1375    truncated: bool,
1376}
1377
1378impl<'a> StrictAvroDatumReader<'a> {
1379    fn new(bytes: &'a [u8]) -> Self {
1380        Self {
1381            bytes,
1382            offset: 0,
1383            truncated: false,
1384        }
1385    }
1386
1387    fn remaining(&self) -> usize {
1388        self.bytes.len() - self.offset
1389    }
1390}
1391
1392impl Read for StrictAvroDatumReader<'_> {
1393    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1394        let count = buffer.len().min(self.remaining());
1395        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1396        self.offset += count;
1397        if count < buffer.len() {
1398            self.truncated = true;
1399        }
1400
1401        Ok(count)
1402    }
1403}
1404
1405fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1406    let branch = match value {
1407        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1408        AvroValue::Boolean(value) => AvroDatum::Union(
1409            1,
1410            Box::new(AvroDatum::Record(vec![(
1411                "boolean".to_string(),
1412                AvroDatum::Boolean(*value),
1413            )])),
1414        ),
1415        AvroValue::Long(value) => AvroDatum::Union(
1416            2,
1417            Box::new(AvroDatum::Record(vec![(
1418                "long".to_string(),
1419                AvroDatum::Long(*value),
1420            )])),
1421        ),
1422        AvroValue::Double(value) => {
1423            if !value.is_finite() {
1424                return Err(Error::Codec(
1425                    "non_finite_float: Avro Value doubles must be finite".to_string(),
1426                ));
1427            }
1428            AvroDatum::Union(
1429                3,
1430                Box::new(AvroDatum::Record(vec![(
1431                    "double".to_string(),
1432                    AvroDatum::Double(*value),
1433                )])),
1434            )
1435        }
1436        AvroValue::Bytes(value) => AvroDatum::Union(
1437            4,
1438            Box::new(AvroDatum::Record(vec![(
1439                "bytes".to_string(),
1440                AvroDatum::Bytes(value.clone()),
1441            )])),
1442        ),
1443        AvroValue::String(value) => AvroDatum::Union(
1444            5,
1445            Box::new(AvroDatum::Record(vec![(
1446                "string".to_string(),
1447                AvroDatum::String(value.clone()),
1448            )])),
1449        ),
1450        AvroValue::Array(values) => AvroDatum::Union(
1451            6,
1452            Box::new(AvroDatum::Record(vec![(
1453                "items".to_string(),
1454                AvroDatum::Array(
1455                    values
1456                        .iter()
1457                        .map(avro_value_to_datum)
1458                        .collect::<Result<Vec<_>>>()?,
1459                ),
1460            )])),
1461        ),
1462        AvroValue::Map(values) => AvroDatum::Union(
1463            7,
1464            Box::new(AvroDatum::Record(vec![(
1465                "entries".to_string(),
1466                AvroDatum::Map(
1467                    values
1468                        .iter()
1469                        .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1470                        .collect::<Result<HashMap<_, _>>>()?,
1471                ),
1472            )])),
1473        ),
1474    };
1475    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1476}
1477
1478fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1479    let AvroDatum::Record(mut outer) = datum else {
1480        return Err(Error::Codec(
1481            "invalid_payload_framing: datum is not a Value record".to_string(),
1482        ));
1483    };
1484    let (_, branch) = outer
1485        .pop()
1486        .filter(|(name, _)| name == "value")
1487        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1488    let AvroDatum::Union(_, branch) = branch else {
1489        return Err(Error::Codec(
1490            "invalid_payload_framing: invalid Value union".to_string(),
1491        ));
1492    };
1493    match *branch {
1494        AvroDatum::Null => Ok(AvroValue::Null),
1495        AvroDatum::Record(mut fields) => {
1496            let (name, value) = fields.pop().ok_or_else(|| {
1497                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1498            })?;
1499            match (name.as_str(), value) {
1500                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1501                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1502                ("double", AvroDatum::Double(value)) if value.is_finite() => {
1503                    Ok(AvroValue::Double(value))
1504                }
1505                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1506                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1507                ("items", AvroDatum::Array(values)) => values
1508                    .into_iter()
1509                    .map(avro_value_from_datum)
1510                    .collect::<Result<Vec<_>>>()
1511                    .map(AvroValue::Array),
1512                ("entries", AvroDatum::Map(values)) => values
1513                    .into_iter()
1514                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1515                    .collect::<Result<BTreeMap<_, _>>>()
1516                    .map(AvroValue::Map),
1517                _ => Err(Error::Codec(
1518                    "invalid_payload_framing: unknown Value branch".to_string(),
1519                )),
1520            }
1521        }
1522        _ => Err(Error::Codec(
1523            "invalid_payload_framing: invalid Value branch".to_string(),
1524        )),
1525    }
1526}
1527
1528fn avro_value_schema() -> Result<&'static Schema> {
1529    match AVRO_VALUE_SCHEMA.get_or_init(|| {
1530        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1531            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1532    }) {
1533        Ok(schema) => Ok(schema),
1534        Err(message) => Err(Error::Codec(message.clone())),
1535    }
1536}
1537
1538#[derive(Clone, Debug)]
1539pub struct Client {
1540    http: reqwest::Client,
1541    base_url: String,
1542    token: Option<String>,
1543    control_token: Option<String>,
1544    worker_token: Option<String>,
1545    namespace: String,
1546}
1547
1548impl Client {
1549    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1550        Self::builder(base_url).build()
1551    }
1552
1553    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1554        ClientBuilder {
1555            base_url: base_url.into(),
1556            token: None,
1557            control_token: None,
1558            worker_token: None,
1559            namespace: "default".to_string(),
1560            timeout: Duration::from_secs(60),
1561        }
1562    }
1563
1564    pub async fn health(&self) -> Result<Value> {
1565        self.request_json(
1566            reqwest::Method::GET,
1567            "/health",
1568            RequestProtocol::ControlPlane,
1569            Option::<&Value>::None,
1570        )
1571        .await
1572    }
1573
1574    pub async fn cluster_info(&self) -> Result<Value> {
1575        self.request_json(
1576            reqwest::Method::GET,
1577            "/cluster/info",
1578            RequestProtocol::ControlPlane,
1579            Option::<&Value>::None,
1580        )
1581        .await
1582    }
1583
1584    pub async fn start_workflow<T: Serialize>(
1585        &self,
1586        workflow_type: &str,
1587        task_queue: &str,
1588        workflow_id: &str,
1589        input: T,
1590    ) -> Result<WorkflowHandle> {
1591        self.start_workflow_with_options(
1592            workflow_type,
1593            task_queue,
1594            workflow_id,
1595            WorkflowStartOptions::default(),
1596            input,
1597        )
1598        .await
1599    }
1600
1601    /// Start a workflow with explicit server-enforced execution and run
1602    /// deadlines.
1603    pub async fn start_workflow_with_options<T: Serialize>(
1604        &self,
1605        workflow_type: &str,
1606        task_queue: &str,
1607        workflow_id: &str,
1608        options: WorkflowStartOptions,
1609        input: T,
1610    ) -> Result<WorkflowHandle> {
1611        options.validate()?;
1612        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1613        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1614        let body = json!({
1615            "workflow_id": workflow_id,
1616            "workflow_type": workflow_type,
1617            "task_queue": task_queue,
1618            "input": input_envelope,
1619            "execution_timeout_seconds": options.execution_timeout_seconds,
1620            "run_timeout_seconds": options.run_timeout_seconds
1621        });
1622
1623        let data: Value = self
1624            .request_json(
1625                reqwest::Method::POST,
1626                "/workflows",
1627                RequestProtocol::ControlPlane,
1628                Some(&body),
1629            )
1630            .await?;
1631
1632        Ok(WorkflowHandle {
1633            client: self.clone(),
1634            workflow_id: data
1635                .get("workflow_id")
1636                .and_then(Value::as_str)
1637                .unwrap_or(workflow_id)
1638                .to_string(),
1639            run_id: data
1640                .get("run_id")
1641                .and_then(Value::as_str)
1642                .map(str::to_string),
1643            workflow_type: data
1644                .get("workflow_type")
1645                .and_then(Value::as_str)
1646                .unwrap_or(workflow_type)
1647                .to_string(),
1648        })
1649    }
1650
1651    pub async fn signal_workflow<T: Serialize>(
1652        &self,
1653        workflow_id: &str,
1654        signal_name: &str,
1655        input: T,
1656    ) -> Result<Value> {
1657        self.signal_workflow_target(workflow_id, None, signal_name, input)
1658            .await
1659    }
1660
1661    /// Signal only if `run_id` is still the current run for this instance.
1662    pub async fn signal_workflow_run<T: Serialize>(
1663        &self,
1664        workflow_id: &str,
1665        run_id: &str,
1666        signal_name: &str,
1667        input: T,
1668    ) -> Result<Value> {
1669        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1670            .await
1671    }
1672
1673    async fn signal_workflow_target<T: Serialize>(
1674        &self,
1675        workflow_id: &str,
1676        run_id: Option<&str>,
1677        signal_name: &str,
1678        input: T,
1679    ) -> Result<Value> {
1680        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1681        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1682        let body = json!({
1683            "input": input_envelope
1684        });
1685        let path = match run_id {
1686            Some(run_id) => {
1687                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1688            }
1689            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1690        };
1691        self.request_json(
1692            reqwest::Method::POST,
1693            &path,
1694            RequestProtocol::ControlPlane,
1695            Some(&body),
1696        )
1697        .await
1698    }
1699
1700    /// Request cooperative cancellation of the current run for an instance.
1701    pub async fn cancel_workflow(
1702        &self,
1703        workflow_id: &str,
1704        options: WorkflowCommandOptions,
1705    ) -> Result<WorkflowCommandResult> {
1706        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1707            .await
1708    }
1709
1710    /// Request cooperative cancellation only if `run_id` is still current.
1711    pub async fn cancel_workflow_run(
1712        &self,
1713        workflow_id: &str,
1714        run_id: &str,
1715        options: WorkflowCommandOptions,
1716    ) -> Result<WorkflowCommandResult> {
1717        self.workflow_command(
1718            workflow_id,
1719            Some(run_id),
1720            WorkflowCommandKind::Cancel,
1721            options,
1722        )
1723        .await
1724    }
1725
1726    /// Forcefully terminate the current run for an instance.
1727    pub async fn terminate_workflow(
1728        &self,
1729        workflow_id: &str,
1730        options: WorkflowCommandOptions,
1731    ) -> Result<WorkflowCommandResult> {
1732        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1733            .await
1734    }
1735
1736    /// Forcefully terminate only if `run_id` is still current.
1737    pub async fn terminate_workflow_run(
1738        &self,
1739        workflow_id: &str,
1740        run_id: &str,
1741        options: WorkflowCommandOptions,
1742    ) -> Result<WorkflowCommandResult> {
1743        self.workflow_command(
1744            workflow_id,
1745            Some(run_id),
1746            WorkflowCommandKind::Terminate,
1747            options,
1748        )
1749        .await
1750    }
1751
1752    async fn workflow_command(
1753        &self,
1754        workflow_id: &str,
1755        run_id: Option<&str>,
1756        command: WorkflowCommandKind,
1757        options: WorkflowCommandOptions,
1758    ) -> Result<WorkflowCommandResult> {
1759        let path = match run_id {
1760            Some(run_id) => format!(
1761                "/workflows/{workflow_id}/runs/{run_id}/{}",
1762                command.as_str()
1763            ),
1764            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1765        };
1766        let data = match self
1767            .request_json(
1768                reqwest::Method::POST,
1769                &path,
1770                RequestProtocol::ControlPlane,
1771                Some(&options),
1772            )
1773            .await
1774        {
1775            Ok(data) => data,
1776            Err(Error::Http { status, body }) => {
1777                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1778                    command,
1779                    status,
1780                    body,
1781                    workflow_id,
1782                    run_id,
1783                )));
1784            }
1785            Err(error) => return Err(error),
1786        };
1787
1788        Ok(workflow_command_result(command, data, workflow_id, run_id))
1789    }
1790
1791    /// Execute a named, read-only query against a running or completed workflow.
1792    ///
1793    /// Arguments and results use the platform payload envelope. Server and
1794    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1795    /// reason, HTTP status, and original response body.
1796    pub async fn query_workflow<T: Serialize>(
1797        &self,
1798        workflow_id: &str,
1799        query_name: &str,
1800        input: T,
1801    ) -> Result<Value> {
1802        self.query_workflow_target(workflow_id, None, query_name, input)
1803            .await
1804    }
1805
1806    /// Query only if `run_id` is still current, preventing accidental retargeting.
1807    pub async fn query_workflow_run<T: Serialize>(
1808        &self,
1809        workflow_id: &str,
1810        run_id: &str,
1811        query_name: &str,
1812        input: T,
1813    ) -> Result<Value> {
1814        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1815            .await
1816    }
1817
1818    /// Query a workflow and return the lossless fixed Avro Value result.
1819    pub async fn query_workflow_avro_value<T: Serialize>(
1820        &self,
1821        workflow_id: &str,
1822        query_name: &str,
1823        input: T,
1824    ) -> Result<AvroValue> {
1825        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1826            .await
1827    }
1828
1829    /// Query a selected run and return the lossless fixed Avro Value result.
1830    pub async fn query_workflow_run_avro_value<T: Serialize>(
1831        &self,
1832        workflow_id: &str,
1833        run_id: &str,
1834        query_name: &str,
1835        input: T,
1836    ) -> Result<AvroValue> {
1837        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1838            .await
1839    }
1840
1841    async fn query_workflow_avro_value_target<T: Serialize>(
1842        &self,
1843        workflow_id: &str,
1844        run_id: Option<&str>,
1845        query_name: &str,
1846        input: T,
1847    ) -> Result<AvroValue> {
1848        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1849        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1850        let path = match run_id {
1851            Some(run_id) => {
1852                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1853            }
1854            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1855        };
1856        let response: Value = match self
1857            .request_json(
1858                reqwest::Method::POST,
1859                &path,
1860                RequestProtocol::ControlPlane,
1861                Some(&body),
1862            )
1863            .await
1864        {
1865            Ok(response) => response,
1866            Err(Error::Http { status, body }) => {
1867                return Err(Error::QueryFailed(query_failure(status, body)));
1868            }
1869            Err(error) => return Err(error),
1870        };
1871
1872        let envelope = response
1873            .get("result_envelope")
1874            .filter(|envelope| !envelope.is_null())
1875            .ok_or_else(|| {
1876                Error::Codec(
1877                    "missing_payload_envelope: typed query result requires result_envelope"
1878                        .to_string(),
1879                )
1880            })?;
1881        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1882    }
1883
1884    async fn query_workflow_target<T: Serialize>(
1885        &self,
1886        workflow_id: &str,
1887        run_id: Option<&str>,
1888        query_name: &str,
1889        input: T,
1890    ) -> Result<Value> {
1891        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1892        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1893        let body = json!({
1894            "input": input_envelope
1895        });
1896        let path = match run_id {
1897            Some(run_id) => {
1898                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1899            }
1900            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1901        };
1902        let response: Value = match self
1903            .request_json(
1904                reqwest::Method::POST,
1905                &path,
1906                RequestProtocol::ControlPlane,
1907                Some(&body),
1908            )
1909            .await
1910        {
1911            Ok(response) => response,
1912            Err(Error::Http { status, body }) => {
1913                return Err(Error::QueryFailed(query_failure(status, body)));
1914            }
1915            Err(error) => return Err(error),
1916        };
1917
1918        if let Some(envelope) = response
1919            .get("result_envelope")
1920            .filter(|envelope| !envelope.is_null())
1921        {
1922            return decode_wire_value(envelope, DEFAULT_CODEC);
1923        }
1924
1925        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1926    }
1927
1928    /// Send a synchronous update using fixed Avro Value arguments.
1929    pub async fn update_workflow<T: Serialize>(
1930        &self,
1931        workflow_id: &str,
1932        update_name: &str,
1933        input: T,
1934        request_id: Option<&str>,
1935    ) -> Result<Value> {
1936        let response = self
1937            .update_workflow_response(workflow_id, update_name, input, request_id)
1938            .await?;
1939        if let Some(envelope) = response
1940            .get("result_envelope")
1941            .filter(|envelope| !envelope.is_null())
1942        {
1943            return decode_wire_value(envelope, DEFAULT_CODEC);
1944        }
1945        Ok(response.get("result").cloned().unwrap_or(response))
1946    }
1947
1948    /// Send a synchronous update and retain a bytes-capable Avro result.
1949    pub async fn update_workflow_avro_value<T: Serialize>(
1950        &self,
1951        workflow_id: &str,
1952        update_name: &str,
1953        input: T,
1954        request_id: Option<&str>,
1955    ) -> Result<AvroValue> {
1956        let response = self
1957            .update_workflow_response(workflow_id, update_name, input, request_id)
1958            .await?;
1959        let envelope = response
1960            .get("result_envelope")
1961            .filter(|envelope| !envelope.is_null())
1962            .ok_or_else(|| {
1963                Error::Codec(
1964                    "missing_payload_envelope: typed update result requires result_envelope"
1965                        .to_string(),
1966                )
1967            })?;
1968        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1969    }
1970
1971    async fn update_workflow_response<T: Serialize>(
1972        &self,
1973        workflow_id: &str,
1974        update_name: &str,
1975        input: T,
1976        request_id: Option<&str>,
1977    ) -> Result<Value> {
1978        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1979        let mut body = json!({
1980            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1981            "wait_for": "completed",
1982        });
1983        if let Some(request_id) = request_id {
1984            body["request_id"] = json!(request_id);
1985        }
1986        self.request_json(
1987            reqwest::Method::POST,
1988            &format!("/workflows/{workflow_id}/update/{update_name}"),
1989            RequestProtocol::ControlPlane,
1990            Some(&body),
1991        )
1992        .await
1993    }
1994
1995    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
1996        let path = format!("/workflows/{workflow_id}");
1997        let mut data: WorkflowDescription = self
1998            .request_json(
1999                reqwest::Method::GET,
2000                &path,
2001                RequestProtocol::ControlPlane,
2002                Option::<&Value>::None,
2003            )
2004            .await?;
2005        data.decode_payloads()?;
2006        Ok(data)
2007    }
2008
2009    /// Describe one selected run, including historical terminal runs.
2010    pub async fn describe_workflow_run(
2011        &self,
2012        workflow_id: &str,
2013        run_id: &str,
2014    ) -> Result<WorkflowDescription> {
2015        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2016        let mut data: WorkflowDescription = self
2017            .request_json(
2018                reqwest::Method::GET,
2019                &path,
2020                RequestProtocol::ControlPlane,
2021                Option::<&Value>::None,
2022            )
2023            .await?;
2024        data.decode_payloads()?;
2025        Ok(data)
2026    }
2027
2028    pub async fn register_worker(
2029        &self,
2030        worker_id: &str,
2031        task_queue: &str,
2032        supported_workflow_types: Vec<String>,
2033        supported_activity_types: Vec<String>,
2034        max_concurrent_workflow_tasks: usize,
2035        max_concurrent_activity_tasks: usize,
2036    ) -> Result<RegisterWorkerResponse> {
2037        self.register_worker_with_capabilities(
2038            worker_id,
2039            task_queue,
2040            supported_workflow_types,
2041            supported_activity_types,
2042            max_concurrent_workflow_tasks,
2043            max_concurrent_activity_tasks,
2044            Vec::new(),
2045        )
2046        .await
2047    }
2048
2049    /// Register a worker and explicitly advertise additive worker capabilities.
2050    pub async fn register_worker_with_capabilities(
2051        &self,
2052        worker_id: &str,
2053        task_queue: &str,
2054        supported_workflow_types: Vec<String>,
2055        supported_activity_types: Vec<String>,
2056        max_concurrent_workflow_tasks: usize,
2057        max_concurrent_activity_tasks: usize,
2058        capabilities: Vec<String>,
2059    ) -> Result<RegisterWorkerResponse> {
2060        self.register_worker_with_command_contracts(
2061            worker_id,
2062            task_queue,
2063            supported_workflow_types,
2064            supported_activity_types,
2065            max_concurrent_workflow_tasks,
2066            max_concurrent_activity_tasks,
2067            capabilities,
2068            Value::Object(serde_json::Map::new()),
2069        )
2070        .await
2071    }
2072
2073    /// Register a worker and advertise its named query and update handlers.
2074    #[allow(clippy::too_many_arguments)]
2075    pub async fn register_worker_with_command_contracts(
2076        &self,
2077        worker_id: &str,
2078        task_queue: &str,
2079        supported_workflow_types: Vec<String>,
2080        supported_activity_types: Vec<String>,
2081        max_concurrent_workflow_tasks: usize,
2082        max_concurrent_activity_tasks: usize,
2083        capabilities: Vec<String>,
2084        workflow_command_contracts: Value,
2085    ) -> Result<RegisterWorkerResponse> {
2086        let mut body = json!({
2087            "worker_id": worker_id,
2088            "task_queue": task_queue,
2089            "runtime": "rust",
2090            "sdk_version": SDK_VERSION,
2091            "supported_workflow_types": supported_workflow_types,
2092            "supported_activity_types": supported_activity_types,
2093            "capabilities": capabilities,
2094            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2095            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2096        });
2097        if workflow_command_contracts
2098            .as_object()
2099            .is_some_and(|contracts| !contracts.is_empty())
2100        {
2101            body["workflow_command_contracts"] = workflow_command_contracts;
2102        }
2103
2104        self.request_json(
2105            reqwest::Method::POST,
2106            "/worker/register",
2107            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2108            Some(&body),
2109        )
2110        .await
2111    }
2112
2113    /// Long-poll for an ephemeral, read-only workflow query task.
2114    pub async fn poll_query_task(
2115        &self,
2116        worker_id: &str,
2117        task_queue: &str,
2118        timeout: Duration,
2119    ) -> Result<Option<QueryTask>> {
2120        Ok(self
2121            .poll_query_task_response(worker_id, task_queue, timeout)
2122            .await?
2123            .task)
2124    }
2125
2126    /// Poll a query task while preserving server stop and drain metadata.
2127    pub async fn poll_query_task_response(
2128        &self,
2129        worker_id: &str,
2130        task_queue: &str,
2131        timeout: Duration,
2132    ) -> Result<PollQueryTaskResponse> {
2133        let poll_request_id = unique_request_id("rust-query-poll");
2134        self.poll_query_task_response_with_request_id(
2135            worker_id,
2136            task_queue,
2137            timeout,
2138            &poll_request_id,
2139            1,
2140        )
2141        .await
2142    }
2143
2144    async fn poll_query_task_response_with_request_id(
2145        &self,
2146        worker_id: &str,
2147        task_queue: &str,
2148        timeout: Duration,
2149        poll_request_id: &str,
2150        transport_retries: usize,
2151    ) -> Result<PollQueryTaskResponse> {
2152        let timeout_seconds = long_poll_timeout_seconds(timeout);
2153        let body = json!({
2154            "worker_id": worker_id,
2155            "task_queue": task_queue,
2156            "poll_request_id": poll_request_id,
2157            "timeout_seconds": timeout_seconds,
2158        });
2159        self.poll_request_json(
2160            "/worker/query-tasks/poll",
2161            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2162            &body,
2163            timeout + Duration::from_secs(5),
2164            transport_retries,
2165        )
2166        .await
2167    }
2168
2169    /// Complete a query task without appending workflow history.
2170    pub async fn complete_query_task<T: Serialize>(
2171        &self,
2172        query_task_id: &str,
2173        lease_owner: &str,
2174        query_task_attempt: u64,
2175        result: T,
2176        codec: &str,
2177    ) -> Result<Value> {
2178        let typed_result = AvroValue::from_serialize(&result)?;
2179        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2180        self.complete_query_task_with_envelope(
2181            query_task_id,
2182            lease_owner,
2183            query_task_attempt,
2184            typed_result.into_json()?,
2185            result_envelope,
2186        )
2187        .await
2188    }
2189
2190    async fn complete_query_task_with_envelope(
2191        &self,
2192        query_task_id: &str,
2193        lease_owner: &str,
2194        query_task_attempt: u64,
2195        result: Value,
2196        result_envelope: Value,
2197    ) -> Result<Value> {
2198        let body = json!({
2199            "lease_owner": lease_owner,
2200            "query_task_attempt": query_task_attempt,
2201            "result": result,
2202            "result_envelope": result_envelope,
2203        });
2204        let path = format!("/worker/query-tasks/{query_task_id}/complete");
2205        let response = self
2206            .request_json(
2207                reqwest::Method::POST,
2208                &path,
2209                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2210                Some(&body),
2211            )
2212            .await;
2213        query_task_response(response)
2214    }
2215
2216    /// Report a stable machine-readable query-task failure.
2217    pub async fn fail_query_task(
2218        &self,
2219        query_task_id: &str,
2220        lease_owner: &str,
2221        query_task_attempt: u64,
2222        message: impl Into<String>,
2223        reason: impl Into<String>,
2224        failure_type: impl Into<String>,
2225    ) -> Result<Value> {
2226        let body = json!({
2227            "lease_owner": lease_owner,
2228            "query_task_attempt": query_task_attempt,
2229            "failure": {
2230                "message": message.into(),
2231                "reason": reason.into(),
2232                "type": failure_type.into(),
2233            }
2234        });
2235        let path = format!("/worker/query-tasks/{query_task_id}/fail");
2236        let response = self
2237            .request_json(
2238                reqwest::Method::POST,
2239                &path,
2240                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2241                Some(&body),
2242            )
2243            .await;
2244        query_task_response(response)
2245    }
2246
2247    pub async fn heartbeat_worker(
2248        &self,
2249        worker_id: &str,
2250        workflow_available: usize,
2251        activity_available: usize,
2252    ) -> Result<Value> {
2253        let body = json!({
2254            "worker_id": worker_id,
2255            "task_slots": {
2256                "workflow_available": workflow_available,
2257                "activity_available": activity_available
2258            },
2259            "process_metrics": {
2260                "process_id": std::process::id(),
2261                "process_uptime_seconds": 0
2262            }
2263        });
2264
2265        self.request_json(
2266            reqwest::Method::POST,
2267            "/worker/heartbeat",
2268            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2269            Some(&body),
2270        )
2271        .await
2272    }
2273
2274    pub async fn poll_workflow_task(
2275        &self,
2276        worker_id: &str,
2277        task_queue: &str,
2278        timeout: Duration,
2279    ) -> Result<Option<WorkflowTask>> {
2280        Ok(self
2281            .poll_workflow_task_response(worker_id, task_queue, timeout)
2282            .await?
2283            .task)
2284    }
2285
2286    pub async fn poll_workflow_task_response(
2287        &self,
2288        worker_id: &str,
2289        task_queue: &str,
2290        timeout: Duration,
2291    ) -> Result<PollWorkflowTaskResponse> {
2292        let poll_request_id = unique_request_id("rust-workflow-poll");
2293        self.poll_workflow_task_response_with_request_id(
2294            worker_id,
2295            task_queue,
2296            timeout,
2297            &poll_request_id,
2298            1,
2299        )
2300        .await
2301    }
2302
2303    async fn poll_workflow_task_response_with_request_id(
2304        &self,
2305        worker_id: &str,
2306        task_queue: &str,
2307        timeout: Duration,
2308        poll_request_id: &str,
2309        transport_retries: usize,
2310    ) -> Result<PollWorkflowTaskResponse> {
2311        let body = json!({
2312            "worker_id": worker_id,
2313            "task_queue": task_queue,
2314            "poll_request_id": poll_request_id,
2315            "timeout_seconds": long_poll_timeout_seconds(timeout),
2316        });
2317        let mut data: PollWorkflowTaskResponse = self
2318            .poll_request_json(
2319                "/worker/workflow-tasks/poll",
2320                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2321                &body,
2322                timeout + Duration::from_secs(5),
2323                transport_retries,
2324            )
2325            .await?;
2326
2327        if let Some(task) = data.task.as_mut() {
2328            self.fetch_remaining_workflow_history(worker_id, task)
2329                .await?;
2330        }
2331
2332        Ok(data)
2333    }
2334
2335    async fn fetch_remaining_workflow_history(
2336        &self,
2337        worker_id: &str,
2338        task: &mut WorkflowTask,
2339    ) -> Result<()> {
2340        let mut next_token = task.next_history_page_token.clone();
2341
2342        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2343            let lease_owner = task
2344                .lease_owner
2345                .clone()
2346                .unwrap_or_else(|| worker_id.to_string());
2347            let page = self
2348                .workflow_task_history_page(
2349                    &task.task_id,
2350                    &lease_owner,
2351                    task.workflow_task_attempt,
2352                    &token,
2353                )
2354                .await?;
2355
2356            task.append_history_page(page);
2357
2358            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2359                return Err(Error::Codec(
2360                    "workflow history pagination returned the same page token".to_string(),
2361                ));
2362            }
2363
2364            next_token = task.next_history_page_token.clone();
2365        }
2366
2367        Ok(())
2368    }
2369
2370    async fn workflow_task_history_page(
2371        &self,
2372        task_id: &str,
2373        lease_owner: &str,
2374        workflow_task_attempt: u64,
2375        next_history_page_token: &str,
2376    ) -> Result<WorkflowTaskHistoryPage> {
2377        let body = json!({
2378            "lease_owner": lease_owner,
2379            "workflow_task_attempt": workflow_task_attempt,
2380            "next_history_page_token": next_history_page_token
2381        });
2382        let path = format!("/worker/workflow-tasks/{task_id}/history");
2383
2384        self.request_json(
2385            reqwest::Method::POST,
2386            &path,
2387            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2388            Some(&body),
2389        )
2390        .await
2391    }
2392
2393    pub async fn complete_workflow_task(
2394        &self,
2395        task_id: &str,
2396        lease_owner: &str,
2397        workflow_task_attempt: u64,
2398        commands: Vec<Value>,
2399    ) -> Result<Value> {
2400        let body = json!({
2401            "lease_owner": lease_owner,
2402            "workflow_task_attempt": workflow_task_attempt,
2403            "commands": commands
2404        });
2405        let path = format!("/worker/workflow-tasks/{task_id}/complete");
2406        self.request_json(
2407            reqwest::Method::POST,
2408            &path,
2409            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2410            Some(&body),
2411        )
2412        .await
2413    }
2414
2415    pub async fn fail_workflow_task(
2416        &self,
2417        task_id: &str,
2418        lease_owner: &str,
2419        workflow_task_attempt: u64,
2420        message: impl Into<String>,
2421    ) -> Result<Value> {
2422        self.fail_workflow_task_with_type(
2423            task_id,
2424            lease_owner,
2425            workflow_task_attempt,
2426            message,
2427            "RustWorkflowTaskFailure",
2428        )
2429        .await
2430    }
2431
2432    async fn fail_workflow_task_with_type(
2433        &self,
2434        task_id: &str,
2435        lease_owner: &str,
2436        workflow_task_attempt: u64,
2437        message: impl Into<String>,
2438        failure_type: &str,
2439    ) -> Result<Value> {
2440        let body = json!({
2441            "lease_owner": lease_owner,
2442            "workflow_task_attempt": workflow_task_attempt,
2443            "failure": {
2444                "message": message.into(),
2445                "type": failure_type
2446            }
2447        });
2448        let path = format!("/worker/workflow-tasks/{task_id}/fail");
2449        self.request_json(
2450            reqwest::Method::POST,
2451            &path,
2452            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2453            Some(&body),
2454        )
2455        .await
2456    }
2457
2458    pub async fn poll_activity_task(
2459        &self,
2460        worker_id: &str,
2461        task_queue: &str,
2462        timeout: Duration,
2463    ) -> Result<Option<ActivityTask>> {
2464        Ok(self
2465            .poll_activity_task_response(worker_id, task_queue, timeout)
2466            .await?
2467            .task)
2468    }
2469
2470    /// Poll an activity task while preserving server stop and drain metadata.
2471    pub async fn poll_activity_task_response(
2472        &self,
2473        worker_id: &str,
2474        task_queue: &str,
2475        timeout: Duration,
2476    ) -> Result<PollActivityTaskResponse> {
2477        let poll_request_id = unique_request_id("rust-activity-poll");
2478        self.poll_activity_task_response_with_request_id(
2479            worker_id,
2480            task_queue,
2481            timeout,
2482            &poll_request_id,
2483            1,
2484        )
2485        .await
2486    }
2487
2488    async fn poll_activity_task_response_with_request_id(
2489        &self,
2490        worker_id: &str,
2491        task_queue: &str,
2492        timeout: Duration,
2493        poll_request_id: &str,
2494        transport_retries: usize,
2495    ) -> Result<PollActivityTaskResponse> {
2496        let body = json!({
2497            "worker_id": worker_id,
2498            "task_queue": task_queue,
2499            "poll_request_id": poll_request_id,
2500            "timeout_seconds": long_poll_timeout_seconds(timeout),
2501        });
2502        let data: PollActivityTaskResponse = self
2503            .poll_request_json(
2504                "/worker/activity-tasks/poll",
2505                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2506                &body,
2507                timeout + Duration::from_secs(5),
2508                transport_retries,
2509            )
2510            .await?;
2511        Ok(data)
2512    }
2513
2514    pub async fn complete_activity_task<T: Serialize>(
2515        &self,
2516        task_id: &str,
2517        activity_attempt_id: &str,
2518        lease_owner: &str,
2519        result: T,
2520        codec: &str,
2521    ) -> Result<Value> {
2522        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2523        let body = json!({
2524            "activity_attempt_id": activity_attempt_id,
2525            "lease_owner": lease_owner,
2526            "result": result
2527        });
2528        let path = format!("/worker/activity-tasks/{task_id}/complete");
2529        activity_task_response(
2530            self.request_json(
2531                reqwest::Method::POST,
2532                &path,
2533                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2534                Some(&body),
2535            )
2536            .await,
2537            "complete",
2538            task_id,
2539            activity_attempt_id,
2540        )
2541    }
2542
2543    pub async fn fail_activity_task(
2544        &self,
2545        task_id: &str,
2546        activity_attempt_id: &str,
2547        lease_owner: &str,
2548        message: impl Into<String>,
2549        non_retryable: bool,
2550    ) -> Result<Value> {
2551        let body = json!({
2552            "activity_attempt_id": activity_attempt_id,
2553            "lease_owner": lease_owner,
2554            "failure": {
2555                "message": message.into(),
2556                "type": "RustActivityFailure",
2557                "non_retryable": non_retryable
2558            }
2559        });
2560        let path = format!("/worker/activity-tasks/{task_id}/fail");
2561        activity_task_response(
2562            self.request_json(
2563                reqwest::Method::POST,
2564                &path,
2565                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2566                Some(&body),
2567            )
2568            .await,
2569            "fail",
2570            task_id,
2571            activity_attempt_id,
2572        )
2573    }
2574
2575    pub async fn heartbeat_activity_task<T: Serialize>(
2576        &self,
2577        task_id: &str,
2578        activity_attempt_id: &str,
2579        lease_owner: &str,
2580        details: T,
2581    ) -> Result<ActivityHeartbeatResponse> {
2582        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2583        let body = json!({
2584            "activity_attempt_id": activity_attempt_id,
2585            "lease_owner": lease_owner,
2586            "details": details
2587        });
2588        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2589        activity_task_response(
2590            self.request_json(
2591                reqwest::Method::POST,
2592                &path,
2593                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2594                Some(&body),
2595            )
2596            .await,
2597            "heartbeat",
2598            task_id,
2599            activity_attempt_id,
2600        )
2601    }
2602
2603    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2604        &self,
2605        method: reqwest::Method,
2606        path: &str,
2607        protocol: RequestProtocol,
2608        body: Option<&B>,
2609    ) -> Result<T> {
2610        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2611            .await
2612    }
2613
2614    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2615        &self,
2616        method: reqwest::Method,
2617        path: &str,
2618        protocol: RequestProtocol,
2619        body: Option<&B>,
2620        timeout: Duration,
2621    ) -> Result<T> {
2622        let mut request = self
2623            .http
2624            .request(method, format!("{}/api{}", self.base_url, path))
2625            .timeout(timeout)
2626            .header(reqwest::header::ACCEPT, "application/json")
2627            .header(reqwest::header::CONTENT_TYPE, "application/json")
2628            .header("X-Namespace", &self.namespace);
2629
2630        match protocol {
2631            RequestProtocol::Worker(version) => {
2632                request = request.header("X-Durable-Workflow-Protocol-Version", version);
2633            }
2634            RequestProtocol::ControlPlane => {
2635                request = request.header(
2636                    "X-Durable-Workflow-Control-Plane-Version",
2637                    CONTROL_PLANE_VERSION,
2638                );
2639            }
2640        }
2641
2642        if let Some(token) = self.auth_token(protocol.is_worker()) {
2643            request = request.bearer_auth(token);
2644        }
2645
2646        if let Some(body) = body {
2647            request = request.json(body);
2648        }
2649
2650        let response = request.send().await?;
2651        let status = response.status();
2652        let bytes = response.bytes().await?;
2653
2654        if !status.is_success() {
2655            let body = String::from_utf8_lossy(&bytes).to_string();
2656            if let Some(protocol) = protocol_failure(status, &body) {
2657                return Err(Error::Protocol(protocol));
2658            }
2659            return Err(Error::Http { status, body });
2660        }
2661
2662        if bytes.is_empty() {
2663            return Ok(serde_json::from_value(Value::Null)?);
2664        }
2665
2666        Ok(serde_json::from_slice(&bytes)?)
2667    }
2668
2669    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2670        &self,
2671        path: &str,
2672        protocol: RequestProtocol,
2673        body: &B,
2674        timeout: Duration,
2675        max_retries: usize,
2676    ) -> Result<T> {
2677        let mut retries = 0;
2678
2679        loop {
2680            let response = self
2681                .request_json_with_timeout(
2682                    reqwest::Method::POST,
2683                    path,
2684                    protocol,
2685                    Some(body),
2686                    timeout,
2687                )
2688                .await;
2689
2690            match response {
2691                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2692                response => return worker_poll_response(response),
2693            }
2694        }
2695    }
2696
2697    fn auth_token(&self, worker: bool) -> Option<&str> {
2698        if worker {
2699            self.worker_token
2700                .as_deref()
2701                .or(self.token.as_deref())
2702                .or(self.control_token.as_deref())
2703        } else {
2704            self.control_token
2705                .as_deref()
2706                .or(self.token.as_deref())
2707                .or(self.worker_token.as_deref())
2708        }
2709    }
2710}
2711
2712fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2713    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2714    let reason = body
2715        .get("reason")
2716        .and_then(Value::as_str)
2717        .unwrap_or("query_rejected")
2718        .to_string();
2719    let message = body
2720        .get("message")
2721        .or_else(|| body.get("error"))
2722        .and_then(Value::as_str)
2723        .unwrap_or("workflow query was rejected")
2724        .to_string();
2725
2726    QueryFailure {
2727        status: status.as_u16(),
2728        reason,
2729        message,
2730        body,
2731    }
2732}
2733
2734fn workflow_command_result(
2735    command: WorkflowCommandKind,
2736    data: Value,
2737    workflow_id: &str,
2738    run_id: Option<&str>,
2739) -> WorkflowCommandResult {
2740    WorkflowCommandResult {
2741        command,
2742        workflow_id: data
2743            .get("workflow_id")
2744            .and_then(Value::as_str)
2745            .unwrap_or(workflow_id)
2746            .to_string(),
2747        run_id: data
2748            .get("run_id")
2749            .and_then(Value::as_str)
2750            .or(run_id)
2751            .map(str::to_string),
2752        outcome: data
2753            .get("outcome")
2754            .and_then(Value::as_str)
2755            .map(str::to_string),
2756        reason: data
2757            .get("reason")
2758            .and_then(Value::as_str)
2759            .map(str::to_string),
2760        command_status: data
2761            .get("command_status")
2762            .and_then(Value::as_str)
2763            .map(str::to_string),
2764        raw: data,
2765    }
2766}
2767
2768fn workflow_command_rejection(
2769    command: WorkflowCommandKind,
2770    status: reqwest::StatusCode,
2771    raw_body: String,
2772    workflow_id: &str,
2773    run_id: Option<&str>,
2774) -> WorkflowCommandRejection {
2775    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2776    WorkflowCommandRejection {
2777        command,
2778        status: status.as_u16(),
2779        reason: body
2780            .get("reason")
2781            .and_then(Value::as_str)
2782            .unwrap_or("workflow_command_rejected")
2783            .to_string(),
2784        message: body
2785            .get("message")
2786            .or_else(|| body.get("error"))
2787            .and_then(Value::as_str)
2788            .unwrap_or("workflow lifecycle command was rejected")
2789            .to_string(),
2790        workflow_id: body
2791            .get("workflow_id")
2792            .and_then(Value::as_str)
2793            .unwrap_or(workflow_id)
2794            .to_string(),
2795        run_id: body
2796            .get("run_id")
2797            .and_then(Value::as_str)
2798            .or(run_id)
2799            .map(str::to_string),
2800        target_scope: body
2801            .get("target_scope")
2802            .and_then(Value::as_str)
2803            .map(str::to_string),
2804        body,
2805    }
2806}
2807
2808fn query_task_response(response: Result<Value>) -> Result<Value> {
2809    match response {
2810        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2811        response => response,
2812    }
2813}
2814
2815fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2816    match response {
2817        Err(Error::Http { status, body })
2818            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2819        {
2820            Ok(serde_json::from_str(&body)?)
2821        }
2822        response => response,
2823    }
2824}
2825
2826fn worker_poll_body_is_stop(body: &str) -> bool {
2827    serde_json::from_str::<Value>(body)
2828        .ok()
2829        .is_some_and(|body| {
2830            worker_poll_is_stop(
2831                body.get("poll_status").and_then(Value::as_str),
2832                body.get("reason").and_then(Value::as_str),
2833            )
2834        })
2835}
2836
2837fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2838    matches!(poll_status, Some("draining" | "stopped"))
2839        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2840}
2841
2842fn query_task_rejection_is_final(error: &Error) -> bool {
2843    matches!(
2844        error,
2845        Error::QueryFailed(failure)
2846            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2847    )
2848}
2849
2850fn activity_task_response<T>(
2851    response: Result<T>,
2852    operation: &str,
2853    task_id: &str,
2854    activity_attempt_id: &str,
2855) -> Result<T> {
2856    match response {
2857        Err(Error::Http { status, body }) => {
2858            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2859            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2860                operation: operation.to_string(),
2861                status: status.as_u16(),
2862                reason: body
2863                    .get("reason")
2864                    .and_then(Value::as_str)
2865                    .unwrap_or("activity_task_rejected")
2866                    .to_string(),
2867                task_id: body
2868                    .get("task_id")
2869                    .and_then(Value::as_str)
2870                    .unwrap_or(task_id)
2871                    .to_string(),
2872                activity_attempt_id: body
2873                    .get("activity_attempt_id")
2874                    .and_then(Value::as_str)
2875                    .unwrap_or(activity_attempt_id)
2876                    .to_string(),
2877                cancel_requested: body
2878                    .get("cancel_requested")
2879                    .and_then(Value::as_bool)
2880                    .unwrap_or(false),
2881                can_continue: body.get("can_continue").and_then(Value::as_bool),
2882                run_closed_reason: body
2883                    .get("run_closed_reason")
2884                    .and_then(Value::as_str)
2885                    .map(str::to_string),
2886                body,
2887            }))
2888        }
2889        response => response,
2890    }
2891}
2892
2893fn activity_task_rejection_is_final(error: &Error) -> bool {
2894    matches!(
2895        error,
2896        Error::ActivityTaskRejected(rejection)
2897            if matches!(
2898                rejection.reason.as_str(),
2899                "run_cancelled"
2900                    | "run_terminated"
2901                    | "attempt_closed"
2902                    | "stale_attempt"
2903                    | "activity_cancelled"
2904                    | "task_cancelled"
2905                    | "run_closed"
2906                    | "activity_not_running"
2907                    | "attempt_not_found"
2908            )
2909    )
2910}
2911
2912fn workflow_task_completion_is_terminal_timeout(
2913    error: &Error,
2914    task_id: &str,
2915    workflow_task_attempt: u64,
2916    run_id: Option<&str>,
2917) -> bool {
2918    let Error::Http { status, body } = error else {
2919        return false;
2920    };
2921    if *status != reqwest::StatusCode::CONFLICT {
2922        return false;
2923    }
2924
2925    let Some(run_id) = run_id else {
2926        return false;
2927    };
2928    let Ok(body) = serde_json::from_str::<Value>(body) else {
2929        return false;
2930    };
2931
2932    body.get("recorded").and_then(Value::as_bool) == Some(false)
2933        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2934        && body.get("run_status").and_then(Value::as_str) == Some("failed")
2935        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2936        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2937        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2938}
2939
2940fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2941    let body: Value = serde_json::from_str(raw_body).ok()?;
2942    let reason = body.get("reason")?.as_str()?;
2943    if !matches!(
2944        reason,
2945        "missing_protocol_version"
2946            | "unsupported_protocol_version"
2947            | "missing_control_plane_version"
2948            | "unsupported_control_plane_version"
2949    ) {
2950        return None;
2951    }
2952
2953    Some(ProtocolFailure {
2954        status: status.as_u16(),
2955        reason: reason.to_string(),
2956        message: body
2957            .get("message")
2958            .or_else(|| body.get("error"))
2959            .and_then(Value::as_str)
2960            .unwrap_or("protocol version rejected")
2961            .to_string(),
2962        supported_version: body
2963            .get("supported_version")
2964            .and_then(Value::as_str)
2965            .map(str::to_string),
2966        requested_version: body
2967            .get("requested_version")
2968            .and_then(Value::as_str)
2969            .map(str::to_string),
2970        body,
2971    })
2972}
2973
2974fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2975    timeout
2976        .as_secs()
2977        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2978        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2979}
2980
2981fn worker_operation_is_retryable(error: &Error) -> bool {
2982    match error {
2983        Error::Transport(error) => {
2984            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2985        }
2986        Error::Http { status, .. } => {
2987            matches!(
2988                *status,
2989                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2990            ) || status.is_server_error()
2991        }
2992        _ => false,
2993    }
2994}
2995
2996fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
2997    let exponent = retry.saturating_sub(1).min(31) as u32;
2998    policy
2999        .initial_backoff
3000        .saturating_mul(1_u32 << exponent)
3001        .min(policy.max_backoff)
3002}
3003
3004#[derive(Debug)]
3005pub struct ClientBuilder {
3006    base_url: String,
3007    token: Option<String>,
3008    control_token: Option<String>,
3009    worker_token: Option<String>,
3010    namespace: String,
3011    timeout: Duration,
3012}
3013
3014impl ClientBuilder {
3015    pub fn token(mut self, token: Option<String>) -> Self {
3016        self.token = token;
3017        self
3018    }
3019
3020    pub fn control_token(mut self, token: Option<String>) -> Self {
3021        self.control_token = token;
3022        self
3023    }
3024
3025    pub fn worker_token(mut self, token: Option<String>) -> Self {
3026        self.worker_token = token;
3027        self
3028    }
3029
3030    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3031        self.namespace = namespace.into();
3032        self
3033    }
3034
3035    pub fn timeout(mut self, timeout: Duration) -> Self {
3036        self.timeout = timeout;
3037        self
3038    }
3039
3040    pub fn build(self) -> Result<Client> {
3041        Ok(Client {
3042            http: reqwest::Client::builder().timeout(self.timeout).build()?,
3043            base_url: self.base_url.trim_end_matches('/').to_string(),
3044            token: self.token,
3045            control_token: self.control_token,
3046            worker_token: self.worker_token,
3047            namespace: self.namespace,
3048        })
3049    }
3050}
3051
3052#[derive(Clone, Debug)]
3053pub struct WorkflowHandle {
3054    client: Client,
3055    pub workflow_id: String,
3056    pub run_id: Option<String>,
3057    pub workflow_type: String,
3058}
3059
3060impl WorkflowHandle {
3061    /// Describe whichever run is current for this stable workflow instance.
3062    pub async fn describe(&self) -> Result<WorkflowDescription> {
3063        self.client.describe_workflow(&self.workflow_id).await
3064    }
3065
3066    /// Describe the run identity originally selected by this handle.
3067    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3068        let run_id = self.run_id.as_deref().ok_or_else(|| {
3069            Error::Codec("run_id is required for selected-run description".to_string())
3070        })?;
3071        self.client
3072            .describe_workflow_run(&self.workflow_id, run_id)
3073            .await
3074    }
3075
3076    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3077        self.client
3078            .signal_workflow(&self.workflow_id, signal_name, input)
3079            .await
3080    }
3081
3082    /// Signal only if this handle's selected run is still current.
3083    pub async fn signal_selected_run<T: Serialize>(
3084        &self,
3085        signal_name: &str,
3086        input: T,
3087    ) -> Result<Value> {
3088        let run_id = self.run_id.as_deref().ok_or_else(|| {
3089            Error::Codec("run_id is required for selected-run signaling".to_string())
3090        })?;
3091        self.client
3092            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3093            .await
3094    }
3095
3096    /// Request cooperative cancellation of whichever run is current.
3097    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3098        self.client
3099            .cancel_workflow(&self.workflow_id, options)
3100            .await
3101    }
3102
3103    /// Request cancellation only if this handle's selected run is still current.
3104    pub async fn cancel_selected_run(
3105        &self,
3106        options: WorkflowCommandOptions,
3107    ) -> Result<WorkflowCommandResult> {
3108        let run_id = self.run_id.as_deref().ok_or_else(|| {
3109            Error::Codec("run_id is required for selected-run cancellation".to_string())
3110        })?;
3111        self.client
3112            .cancel_workflow_run(&self.workflow_id, run_id, options)
3113            .await
3114    }
3115
3116    /// Forcefully terminate whichever run is current.
3117    pub async fn terminate(
3118        &self,
3119        options: WorkflowCommandOptions,
3120    ) -> Result<WorkflowCommandResult> {
3121        self.client
3122            .terminate_workflow(&self.workflow_id, options)
3123            .await
3124    }
3125
3126    /// Terminate only if this handle's selected run is still current.
3127    pub async fn terminate_selected_run(
3128        &self,
3129        options: WorkflowCommandOptions,
3130    ) -> Result<WorkflowCommandResult> {
3131        let run_id = self.run_id.as_deref().ok_or_else(|| {
3132            Error::Codec("run_id is required for selected-run termination".to_string())
3133        })?;
3134        self.client
3135            .terminate_workflow_run(&self.workflow_id, run_id, options)
3136            .await
3137    }
3138
3139    /// Execute a named, read-only query against this workflow.
3140    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3141        self.client
3142            .query_workflow(&self.workflow_id, query_name, input)
3143            .await
3144    }
3145
3146    pub async fn query_avro_value<T: Serialize>(
3147        &self,
3148        query_name: &str,
3149        input: T,
3150    ) -> Result<AvroValue> {
3151        self.client
3152            .query_workflow_avro_value(&self.workflow_id, query_name, input)
3153            .await
3154    }
3155
3156    pub async fn update<T: Serialize>(
3157        &self,
3158        update_name: &str,
3159        input: T,
3160        request_id: Option<&str>,
3161    ) -> Result<Value> {
3162        self.client
3163            .update_workflow(&self.workflow_id, update_name, input, request_id)
3164            .await
3165    }
3166
3167    pub async fn update_avro_value<T: Serialize>(
3168        &self,
3169        update_name: &str,
3170        input: T,
3171        request_id: Option<&str>,
3172    ) -> Result<AvroValue> {
3173        self.client
3174            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3175            .await
3176    }
3177
3178    /// Query only if this handle's selected run is still current.
3179    pub async fn query_selected_run<T: Serialize>(
3180        &self,
3181        query_name: &str,
3182        input: T,
3183    ) -> Result<Value> {
3184        let run_id = self
3185            .run_id
3186            .as_deref()
3187            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3188        self.client
3189            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3190            .await
3191    }
3192
3193    /// Await the final terminal outcome of the current continue-as-new chain.
3194    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3195        self.result_target(options, None).await
3196    }
3197
3198    /// Await the final result without projecting Avro bytes through JSON.
3199    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3200        self.result_avro_value_target(options, None).await
3201    }
3202
3203    /// Await only the run identity originally selected by this handle.
3204    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3205        let run_id = self.run_id.as_deref().ok_or_else(|| {
3206            Error::Codec("run_id is required for selected-run result".to_string())
3207        })?;
3208        self.result_target(options, Some(run_id)).await
3209    }
3210
3211    /// Await the selected run's result on the lossless Avro Value surface.
3212    pub async fn result_selected_run_avro_value(
3213        &self,
3214        options: WorkflowResultOptions,
3215    ) -> Result<AvroValue> {
3216        let run_id = self.run_id.as_deref().ok_or_else(|| {
3217            Error::Codec("run_id is required for selected-run result".to_string())
3218        })?;
3219        self.result_avro_value_target(options, Some(run_id)).await
3220    }
3221
3222    async fn result_avro_value_target(
3223        &self,
3224        options: WorkflowResultOptions,
3225        selected_run_id: Option<&str>,
3226    ) -> Result<AvroValue> {
3227        let started = Instant::now();
3228
3229        loop {
3230            let description = match selected_run_id {
3231                Some(run_id) => {
3232                    self.client
3233                        .describe_workflow_run(&self.workflow_id, run_id)
3234                        .await?
3235                }
3236                None => self.describe().await?,
3237            };
3238            if description.is_completed() {
3239                return description.output_avro_value.ok_or_else(|| {
3240                    Error::Codec(
3241                        "missing_payload_envelope: typed workflow result requires output_envelope"
3242                            .to_string(),
3243                    )
3244                });
3245            }
3246            if description.is_terminal() {
3247                let outcome =
3248                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3249                return Err(match outcome.kind {
3250                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3251                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3252                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3253                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3254                });
3255            }
3256            if started.elapsed() >= options.timeout {
3257                return Err(Error::Timeout);
3258            }
3259            tokio::time::sleep(options.poll_interval).await;
3260        }
3261    }
3262
3263    async fn result_target(
3264        &self,
3265        options: WorkflowResultOptions,
3266        selected_run_id: Option<&str>,
3267    ) -> Result<Value> {
3268        let started = Instant::now();
3269
3270        loop {
3271            let description = match selected_run_id {
3272                Some(run_id) => {
3273                    self.client
3274                        .describe_workflow_run(&self.workflow_id, run_id)
3275                        .await?
3276                }
3277                None => self.describe().await?,
3278            };
3279            if description.is_completed() {
3280                return Ok(description.output.unwrap_or(Value::Null));
3281            }
3282
3283            if description.is_terminal() {
3284                let outcome =
3285                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3286                return Err(match outcome.kind {
3287                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3288                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3289                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3290                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3291                });
3292            }
3293
3294            if started.elapsed() >= options.timeout {
3295                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3296                    kind: WorkflowTerminalKind::TimedOut,
3297                    workflow_id: description
3298                        .workflow_id
3299                        .clone()
3300                        .unwrap_or_else(|| self.workflow_id.clone()),
3301                    run_id: description
3302                        .run_id
3303                        .clone()
3304                        .or_else(|| selected_run_id.map(str::to_string)),
3305                    reason: "result_wait_timeout".to_string(),
3306                    failure_category: Some("client_timeout".to_string()),
3307                    failure_id: None,
3308                    exception_type: None,
3309                    exception_class: None,
3310                    non_retryable: None,
3311                    message: Some(format!(
3312                        "workflow result was not terminal within {:?}",
3313                        options.timeout
3314                    )),
3315                    exception: None,
3316                    raw: description.raw_value(),
3317                }));
3318            }
3319
3320            tokio::time::sleep(options.poll_interval).await;
3321        }
3322    }
3323}
3324
3325#[derive(Clone, Copy, Debug)]
3326pub struct WorkflowResultOptions {
3327    pub poll_interval: Duration,
3328    pub timeout: Duration,
3329}
3330
3331impl Default for WorkflowResultOptions {
3332    fn default() -> Self {
3333        Self {
3334            poll_interval: Duration::from_millis(500),
3335            timeout: Duration::from_secs(30),
3336        }
3337    }
3338}
3339
3340#[derive(Clone, Debug, Deserialize)]
3341pub struct WorkflowDescription {
3342    pub workflow_id: Option<String>,
3343    pub run_id: Option<String>,
3344    pub workflow_type: Option<String>,
3345    pub status: Option<String>,
3346    #[serde(default)]
3347    pub closed_reason: Option<String>,
3348    #[serde(default)]
3349    pub error: Option<String>,
3350    #[serde(default)]
3351    pub failure: Option<Value>,
3352    #[serde(default)]
3353    pub exception: Option<Value>,
3354    #[serde(default)]
3355    pub failures: Vec<Value>,
3356    #[serde(default)]
3357    pub output: Option<Value>,
3358    #[serde(default)]
3359    pub output_envelope: Option<Value>,
3360    #[serde(skip)]
3361    pub output_avro_value: Option<AvroValue>,
3362    #[serde(flatten)]
3363    pub raw: HashMap<String, Value>,
3364}
3365
3366impl WorkflowDescription {
3367    pub fn is_completed(&self) -> bool {
3368        matches!(self.status.as_deref(), Some("completed" | "Completed"))
3369    }
3370
3371    pub fn is_terminal(&self) -> bool {
3372        matches!(
3373            self.status.as_deref(),
3374            Some(
3375                "completed"
3376                    | "Completed"
3377                    | "failed"
3378                    | "Failed"
3379                    | "cancelled"
3380                    | "Cancelled"
3381                    | "terminated"
3382                    | "Terminated"
3383                    | "timed_out"
3384                    | "TimedOut",
3385            )
3386        )
3387    }
3388
3389    fn decode_payloads(&mut self) -> Result<()> {
3390        if let Some(envelope) = &self.output_envelope {
3391            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3392            self.output = Some(value.clone().into_json()?);
3393            self.output_avro_value = Some(value);
3394        }
3395
3396        Ok(())
3397    }
3398
3399    fn raw_value(&self) -> Value {
3400        let mut data = self.raw.clone();
3401        data.insert(
3402            "workflow_id".to_string(),
3403            self.workflow_id
3404                .clone()
3405                .map(Value::String)
3406                .unwrap_or(Value::Null),
3407        );
3408        data.insert(
3409            "run_id".to_string(),
3410            self.run_id
3411                .clone()
3412                .map(Value::String)
3413                .unwrap_or(Value::Null),
3414        );
3415        data.insert(
3416            "workflow_type".to_string(),
3417            self.workflow_type
3418                .clone()
3419                .map(Value::String)
3420                .unwrap_or(Value::Null),
3421        );
3422        data.insert(
3423            "status".to_string(),
3424            self.status
3425                .clone()
3426                .map(Value::String)
3427                .unwrap_or(Value::Null),
3428        );
3429        data.insert(
3430            "closed_reason".to_string(),
3431            self.closed_reason
3432                .clone()
3433                .map(Value::String)
3434                .unwrap_or(Value::Null),
3435        );
3436        if let Some(failure) = &self.failure {
3437            data.insert("failure".to_string(), failure.clone());
3438        }
3439        if let Some(exception) = &self.exception {
3440            data.insert("exception".to_string(), exception.clone());
3441        }
3442        Value::Object(data.into_iter().collect())
3443    }
3444}
3445
3446fn workflow_terminal_outcome(
3447    description: &WorkflowDescription,
3448    workflow_id: &str,
3449    run_id: Option<&str>,
3450) -> WorkflowTerminalOutcome {
3451    let terminal_kind = description
3452        .closed_reason
3453        .as_deref()
3454        .or(description.status.as_deref())
3455        .unwrap_or("failed")
3456        .to_ascii_lowercase();
3457    let kind = match terminal_kind.as_str() {
3458        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3459        "terminated" => WorkflowTerminalKind::Terminated,
3460        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3461        _ => WorkflowTerminalKind::Failed,
3462    };
3463    let default_reason = match kind {
3464        WorkflowTerminalKind::Failed => "workflow_failed",
3465        WorkflowTerminalKind::Cancelled => "cancelled",
3466        WorkflowTerminalKind::Terminated => "terminated",
3467        WorkflowTerminalKind::TimedOut => "timed_out",
3468    };
3469    let failure = description
3470        .failure
3471        .as_ref()
3472        .filter(|value| value.is_object());
3473    let nested_failure = failure
3474        .and_then(|value| value.get("failures"))
3475        .and_then(Value::as_array)
3476        .and_then(|failures| failures.last())
3477        .or_else(|| description.failures.last());
3478    let exception = description
3479        .exception
3480        .clone()
3481        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3482        .or_else(|| {
3483            nested_failure
3484                .and_then(|value| value.get("exception_payload"))
3485                .cloned()
3486        });
3487    let string_field = |name: &str| {
3488        failure
3489            .and_then(|value| value.get(name))
3490            .and_then(Value::as_str)
3491            .or_else(|| {
3492                nested_failure
3493                    .and_then(|value| value.get(name))
3494                    .and_then(Value::as_str)
3495            })
3496            .map(str::to_string)
3497    };
3498    let exception_field = |name: &str| {
3499        exception
3500            .as_ref()
3501            .and_then(|value| value.get(name))
3502            .and_then(Value::as_str)
3503            .map(str::to_string)
3504    };
3505    let message = description
3506        .error
3507        .clone()
3508        .or_else(|| string_field("message"))
3509        .or_else(|| exception_field("message"));
3510    let reason = description
3511        .raw
3512        .get("reason")
3513        .and_then(Value::as_str)
3514        .map(str::to_string)
3515        .or_else(|| {
3516            failure
3517                .and_then(|value| value.get("reason"))
3518                .and_then(Value::as_str)
3519                .map(str::to_string)
3520        })
3521        .or_else(|| description.closed_reason.clone())
3522        .unwrap_or_else(|| default_reason.to_string());
3523    let failure_id = string_field("failure_id").or_else(|| {
3524        nested_failure
3525            .and_then(|value| value.get("id"))
3526            .and_then(Value::as_str)
3527            .map(str::to_string)
3528    });
3529
3530    WorkflowTerminalOutcome {
3531        kind,
3532        workflow_id: description
3533            .workflow_id
3534            .clone()
3535            .unwrap_or_else(|| workflow_id.to_string()),
3536        run_id: description
3537            .run_id
3538            .clone()
3539            .or_else(|| run_id.map(str::to_string)),
3540        reason,
3541        failure_category: string_field("failure_category")
3542            .or_else(|| Some(default_reason.to_string())),
3543        failure_id,
3544        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3545        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3546        non_retryable: failure
3547            .and_then(|value| value.get("non_retryable"))
3548            .and_then(Value::as_bool)
3549            .or_else(|| {
3550                nested_failure
3551                    .and_then(|value| value.get("non_retryable"))
3552                    .and_then(Value::as_bool)
3553            }),
3554        message,
3555        exception,
3556        raw: description.raw_value(),
3557    }
3558}
3559
3560#[derive(Clone, Debug, Deserialize)]
3561pub struct RegisterWorkerResponse {
3562    pub worker_id: String,
3563    pub registered: bool,
3564    #[serde(default)]
3565    pub heartbeat_interval_seconds: Option<u64>,
3566    #[serde(default)]
3567    pub protocol_version: Option<String>,
3568    #[serde(default)]
3569    pub server_capabilities: Option<Value>,
3570}
3571
3572#[derive(Clone, Debug, Deserialize)]
3573pub struct PollWorkflowTaskResponse {
3574    #[serde(default)]
3575    pub task: Option<WorkflowTask>,
3576    #[serde(default)]
3577    pub poll_status: Option<String>,
3578    #[serde(default)]
3579    pub reason: Option<String>,
3580    #[serde(default)]
3581    pub protocol_version: Option<String>,
3582    #[serde(default)]
3583    pub server_capabilities: Option<Value>,
3584}
3585
3586impl PollWorkflowTaskResponse {
3587    /// Classify this response without parsing server display text.
3588    pub fn outcome(&self) -> WorkerPollOutcome {
3589        worker_poll_outcome(
3590            self.task.is_some(),
3591            self.poll_status.as_deref(),
3592            self.reason.as_deref(),
3593        )
3594    }
3595}
3596
3597#[derive(Clone, Debug, Deserialize)]
3598pub struct PollActivityTaskResponse {
3599    #[serde(default)]
3600    pub task: Option<ActivityTask>,
3601    #[serde(default)]
3602    pub poll_status: Option<String>,
3603    #[serde(default)]
3604    pub reason: Option<String>,
3605}
3606
3607impl PollActivityTaskResponse {
3608    /// Classify this response without parsing server display text.
3609    pub fn outcome(&self) -> WorkerPollOutcome {
3610        worker_poll_outcome(
3611            self.task.is_some(),
3612            self.poll_status.as_deref(),
3613            self.reason.as_deref(),
3614        )
3615    }
3616}
3617
3618#[derive(Clone, Debug, Deserialize)]
3619pub struct PollQueryTaskResponse {
3620    #[serde(default)]
3621    pub task: Option<QueryTask>,
3622    #[serde(default)]
3623    pub poll_status: Option<String>,
3624    #[serde(default)]
3625    pub reason: Option<String>,
3626}
3627
3628impl PollQueryTaskResponse {
3629    /// Classify this response without parsing server display text.
3630    pub fn outcome(&self) -> WorkerPollOutcome {
3631        worker_poll_outcome(
3632            self.task.is_some(),
3633            self.poll_status.as_deref(),
3634            self.reason.as_deref(),
3635        )
3636    }
3637}
3638
3639/// Stable classification for worker poll responses.
3640#[derive(Clone, Debug, PartialEq, Eq)]
3641pub enum WorkerPollOutcome {
3642    /// A task was leased and is available on the response.
3643    Task,
3644    /// No task was leased, but the worker should continue polling.
3645    Idle {
3646        poll_status: Option<String>,
3647        reason: Option<String>,
3648    },
3649    /// The server asked this worker to stop claiming new work.
3650    Stop {
3651        poll_status: Option<String>,
3652        reason: Option<String>,
3653    },
3654}
3655
3656impl WorkerPollOutcome {
3657    pub fn should_stop(&self) -> bool {
3658        matches!(self, Self::Stop { .. })
3659    }
3660}
3661
3662fn worker_poll_outcome(
3663    has_task: bool,
3664    poll_status: Option<&str>,
3665    reason: Option<&str>,
3666) -> WorkerPollOutcome {
3667    if worker_poll_is_stop(poll_status, reason) {
3668        return WorkerPollOutcome::Stop {
3669            poll_status: poll_status.map(str::to_string),
3670            reason: reason.map(str::to_string),
3671        };
3672    }
3673
3674    if has_task {
3675        WorkerPollOutcome::Task
3676    } else {
3677        WorkerPollOutcome::Idle {
3678            poll_status: poll_status.map(str::to_string),
3679            reason: reason.map(str::to_string),
3680        }
3681    }
3682}
3683
3684/// An ephemeral server-routed query task.
3685#[derive(Clone, Debug, Deserialize)]
3686pub struct QueryTask {
3687    pub query_task_id: String,
3688    #[serde(default = "default_workflow_task_attempt")]
3689    pub query_task_attempt: u64,
3690    #[serde(default)]
3691    pub lease_owner: Option<String>,
3692    #[serde(default)]
3693    pub workflow_id: Option<String>,
3694    #[serde(default)]
3695    pub run_id: Option<String>,
3696    pub workflow_type: String,
3697    pub query_name: String,
3698    #[serde(default = "default_payload_codec")]
3699    pub payload_codec: String,
3700    #[serde(default)]
3701    pub workflow_arguments: Option<Value>,
3702    #[serde(default)]
3703    pub query_arguments: Option<Value>,
3704    #[serde(default)]
3705    pub history_events: Vec<HistoryEvent>,
3706    #[serde(default)]
3707    pub history_export: Option<Value>,
3708    #[serde(default)]
3709    pub run_status: Option<String>,
3710}
3711
3712#[derive(Clone, Debug, Deserialize)]
3713pub struct WorkflowTask {
3714    pub task_id: String,
3715    #[serde(default)]
3716    pub workflow_id: Option<String>,
3717    #[serde(default)]
3718    pub run_id: Option<String>,
3719    pub workflow_type: String,
3720    #[serde(default = "default_payload_codec")]
3721    pub payload_codec: String,
3722    #[serde(default)]
3723    pub arguments: Option<Value>,
3724    #[serde(default)]
3725    pub history_events: Vec<HistoryEvent>,
3726    #[serde(default)]
3727    pub total_history_events: Option<u64>,
3728    #[serde(default)]
3729    pub history_size_bytes: Option<u64>,
3730    #[serde(default)]
3731    pub continue_as_new_recommended: Option<bool>,
3732    #[serde(default)]
3733    pub history_budget_pressure: Option<String>,
3734    #[serde(default)]
3735    pub next_history_page_token: Option<String>,
3736    #[serde(default = "default_workflow_task_attempt")]
3737    pub workflow_task_attempt: u64,
3738    #[serde(default)]
3739    pub workflow_signal_id: Option<String>,
3740    #[serde(default)]
3741    pub signal_name: Option<String>,
3742    #[serde(default)]
3743    pub signal_arguments: Option<Value>,
3744    #[serde(default)]
3745    pub workflow_update_id: Option<String>,
3746    #[serde(default)]
3747    pub update_name: Option<String>,
3748    #[serde(default)]
3749    pub lease_owner: Option<String>,
3750}
3751
3752impl WorkflowTask {
3753    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3754        self.history_events.extend(page.history_events);
3755
3756        if page.total_history_events.is_some() {
3757            self.total_history_events = page.total_history_events;
3758        }
3759
3760        self.next_history_page_token = page
3761            .next_history_page_token
3762            .filter(|token| !token.is_empty());
3763    }
3764}
3765
3766#[derive(Clone, Debug, Deserialize)]
3767struct WorkflowTaskHistoryPage {
3768    #[serde(default)]
3769    history_events: Vec<HistoryEvent>,
3770    #[serde(default)]
3771    total_history_events: Option<u64>,
3772    #[serde(default)]
3773    next_history_page_token: Option<String>,
3774}
3775
3776#[derive(Clone, Debug, Deserialize)]
3777pub struct ActivityTask {
3778    pub task_id: String,
3779    #[serde(default)]
3780    pub activity_attempt_id: Option<String>,
3781    #[serde(default)]
3782    pub attempt_id: Option<String>,
3783    pub activity_type: String,
3784    #[serde(default = "default_payload_codec")]
3785    pub payload_codec: String,
3786    #[serde(default)]
3787    pub arguments: Option<Value>,
3788    #[serde(default = "default_attempt_number")]
3789    pub attempt_number: u64,
3790    #[serde(default)]
3791    pub lease_owner: Option<String>,
3792}
3793
3794#[derive(Clone, Debug, Deserialize)]
3795pub struct HistoryEvent {
3796    #[serde(alias = "type")]
3797    pub event_type: String,
3798    #[serde(default)]
3799    pub payload: Value,
3800    #[serde(flatten)]
3801    pub raw: HashMap<String, Value>,
3802}
3803
3804/// One decoded signal in the committed workflow-history snapshot.
3805#[derive(Clone, Debug, PartialEq)]
3806pub struct QuerySignal {
3807    pub id: Option<String>,
3808    pub name: String,
3809    pub arguments: Vec<Value>,
3810    avro_arguments: Vec<AvroValue>,
3811    pub workflow_sequence: Option<u64>,
3812}
3813
3814impl QuerySignal {
3815    /// Lossless fixed Avro Value arguments for this committed signal.
3816    pub fn arguments_avro_value(&self) -> &[AvroValue] {
3817        &self.avro_arguments
3818    }
3819}
3820
3821/// Immutable state supplied to a registered query handler.
3822///
3823/// This context intentionally exposes no activity, signal-wait, or command
3824/// APIs. Query handlers inspect committed history and return a value; query
3825/// completion does not append an event or advance deterministic execution.
3826#[derive(Clone, Debug)]
3827pub struct QueryContext {
3828    pub workflow_id: Option<String>,
3829    pub run_id: Option<String>,
3830    pub workflow_type: String,
3831    pub run_status: Option<String>,
3832    workflow_input: Value,
3833    workflow_input_avro_value: AvroValue,
3834    history_events: Arc<Vec<HistoryEvent>>,
3835    signal_events: Arc<Vec<QuerySignal>>,
3836}
3837
3838impl QueryContext {
3839    /// The normalized argument list used to start the workflow.
3840    pub fn workflow_input(&self) -> &Value {
3841        &self.workflow_input
3842    }
3843
3844    /// The lossless fixed Avro Value argument list used to start the workflow.
3845    pub fn workflow_input_avro_value(&self) -> &AvroValue {
3846        &self.workflow_input_avro_value
3847    }
3848
3849    /// The immutable committed history used for this query snapshot.
3850    pub fn history_events(&self) -> &[HistoryEvent] {
3851        self.history_events.as_slice()
3852    }
3853
3854    /// All decoded signals in committed workflow order.
3855    pub fn signal_events(&self) -> &[QuerySignal] {
3856        self.signal_events.as_slice()
3857    }
3858
3859    /// Decoded argument lists for each committed signal with `signal_name`.
3860    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3861        self.signal_events
3862            .iter()
3863            .filter(|signal| signal.name == signal_name)
3864            .map(|signal| signal.arguments.clone())
3865            .collect()
3866    }
3867
3868    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
3869    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3870        self.signal_events
3871            .iter()
3872            .filter(|signal| signal.name == signal_name)
3873            .map(|signal| signal.avro_arguments.clone())
3874            .collect()
3875    }
3876}
3877
3878#[derive(Clone, Debug, Deserialize)]
3879pub struct ActivityHeartbeatResponse {
3880    #[serde(default)]
3881    pub cancel_requested: bool,
3882    #[serde(default)]
3883    pub heartbeat_recorded: bool,
3884    #[serde(default)]
3885    pub can_continue: Option<bool>,
3886    #[serde(default)]
3887    pub reason: Option<String>,
3888    #[serde(default)]
3889    pub run_closed_reason: Option<String>,
3890    #[serde(default)]
3891    pub run_closed_at: Option<String>,
3892    #[serde(default)]
3893    pub lease_expires_at: Option<String>,
3894    #[serde(default)]
3895    pub last_heartbeat_at: Option<String>,
3896}
3897
3898impl ActivityHeartbeatResponse {
3899    /// Whether the activity should stop instead of attempting completion.
3900    pub fn should_stop(&self) -> bool {
3901        self.cancel_requested || self.can_continue == Some(false)
3902    }
3903}
3904
3905fn default_payload_codec() -> String {
3906    DEFAULT_CODEC.to_string()
3907}
3908
3909fn default_workflow_task_attempt() -> u64 {
3910    1
3911}
3912
3913fn default_attempt_number() -> u64 {
3914    1
3915}
3916
3917type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3918type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3919type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3920type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3921type ReplayedWorkflowHandler =
3922    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3923type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3924type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3925type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3926type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3927type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3928type ReplayedQueryHandler = Arc<
3929    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3930        + Send
3931        + Sync,
3932>;
3933type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3934
3935struct ReplayedWorkflowInvocation {
3936    future: WorkflowFuture,
3937    snapshot: WorkflowStateSnapshot,
3938}
3939
3940#[derive(Clone)]
3941struct RegisteredWorkflow {
3942    execute: WorkflowHandler,
3943    replay: Option<ReplayedWorkflowHandler>,
3944    state_type: Option<TypeId>,
3945}
3946
3947#[derive(Clone)]
3948enum RegisteredQuery {
3949    Snapshot(QueryHandler),
3950    Replayed {
3951        state_type: TypeId,
3952        handler: ReplayedQueryHandler,
3953    },
3954}
3955
3956#[derive(Clone, Debug)]
3957pub struct WorkerHeartbeatObservation {
3958    pub worker_id: String,
3959    pub task_queue: String,
3960    pub acknowledged_at_unix_millis: u64,
3961    pub acknowledgement: Value,
3962}
3963
3964/// Bounded retry policy for worker poll acquisition and worker heartbeats.
3965///
3966/// Expected empty long polls are normal successful responses. Transport
3967/// failures, HTTP 408/429 responses, and server errors are retried with capped
3968/// exponential backoff. Authentication, protocol, codec, and handler failures
3969/// are never retried by the worker.
3970#[derive(Clone, Copy, Debug)]
3971pub struct WorkerRetryPolicy {
3972    /// Number of retries after the initial request fails.
3973    pub max_retries: usize,
3974    /// Delay before the first retry.
3975    pub initial_backoff: Duration,
3976    /// Maximum delay between retries.
3977    pub max_backoff: Duration,
3978}
3979
3980impl Default for WorkerRetryPolicy {
3981    fn default() -> Self {
3982        Self {
3983            max_retries: 5,
3984            initial_backoff: Duration::from_millis(100),
3985            max_backoff: Duration::from_secs(5),
3986        }
3987    }
3988}
3989
3990#[derive(Clone, Copy, Debug, PartialEq, Eq)]
3991enum ManagedPollOutcome {
3992    Idle,
3993    Handled,
3994    Stop,
3995}
3996
3997#[derive(Clone)]
3998pub struct Worker {
3999    client: Client,
4000    worker_id: String,
4001    task_queue: String,
4002    workflows: HashMap<String, RegisteredWorkflow>,
4003    activities: HashMap<String, ActivityHandler>,
4004    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4005    updates: HashMap<String, HashMap<String, UpdateHandler>>,
4006    max_concurrent_workflow_tasks: usize,
4007    max_concurrent_activity_tasks: usize,
4008    poll_timeout: Duration,
4009    heartbeat_interval: Duration,
4010    retry_policy: WorkerRetryPolicy,
4011    heartbeat_observer: Option<WorkerHeartbeatObserver>,
4012}
4013
4014impl Worker {
4015    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4016        Self {
4017            client,
4018            worker_id: default_worker_id(),
4019            task_queue: task_queue.into(),
4020            workflows: HashMap::new(),
4021            activities: HashMap::new(),
4022            queries: HashMap::new(),
4023            updates: HashMap::new(),
4024            max_concurrent_workflow_tasks: 10,
4025            max_concurrent_activity_tasks: 10,
4026            poll_timeout: Duration::from_secs(30),
4027            heartbeat_interval: Duration::from_secs(60),
4028            retry_policy: WorkerRetryPolicy::default(),
4029            heartbeat_observer: None,
4030        }
4031    }
4032
4033    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4034        self.worker_id = worker_id.into();
4035        self
4036    }
4037
4038    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4039        self.poll_timeout = timeout;
4040        self
4041    }
4042
4043    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4044        self.heartbeat_interval = interval;
4045        self
4046    }
4047
4048    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
4049    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4050        self.retry_policy = policy;
4051        self
4052    }
4053
4054    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4055    where
4056        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4057    {
4058        self.heartbeat_observer = Some(Arc::new(observer));
4059        self
4060    }
4061
4062    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4063        self.max_concurrent_workflow_tasks = count.max(1);
4064        self
4065    }
4066
4067    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4068        self.max_concurrent_activity_tasks = count.max(1);
4069        self
4070    }
4071
4072    /// Register a workflow handler.
4073    ///
4074    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
4075    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
4076    /// while acquiring or decoding a worker task remain worker-operation
4077    /// failures and do not get converted into workflow outcomes.
4078    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4079    where
4080        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4081        Fut: Future<Output = Result<Value>> + Send + 'static,
4082    {
4083        let handler = Arc::new(handler);
4084        self.workflows.insert(
4085            workflow_type.into(),
4086            RegisteredWorkflow {
4087                execute: Arc::new(move |ctx, input| {
4088                    let handler = Arc::clone(&handler);
4089                    Box::pin(async move {
4090                        let result = handler(ctx, input.into_json()?).await?;
4091                        AvroValue::from_serialize(&result)
4092                    })
4093                }),
4094                replay: None,
4095                state_type: None,
4096            },
4097        );
4098    }
4099
4100    /// Register a workflow on the lossless fixed Avro Value surface.
4101    pub fn register_workflow_avro_value<F, Fut>(
4102        &mut self,
4103        workflow_type: impl Into<String>,
4104        handler: F,
4105    ) where
4106        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4107        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4108    {
4109        self.workflows.insert(
4110            workflow_type.into(),
4111            RegisteredWorkflow {
4112                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4113                replay: None,
4114                state_type: None,
4115            },
4116        );
4117    }
4118
4119    /// Register a workflow whose typed instance state can be reconstructed for queries.
4120    ///
4121    /// `state_factory` creates a fresh instance for every normal workflow task and
4122    /// query replay. The workflow handler is the single source of truth for state
4123    /// transitions: it updates [`WorkflowInstance`] after activities and signals
4124    /// resolve. Query replay runs this same handler over committed history and
4125    /// discards any commands it would emit.
4126    pub fn register_replayed_workflow<S, Factory, F, Fut>(
4127        &mut self,
4128        workflow_type: impl Into<String>,
4129        state_factory: Factory,
4130        handler: F,
4131    ) where
4132        S: Clone + Send + Sync + 'static,
4133        Factory: Fn() -> S + Send + Sync + 'static,
4134        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4135        Fut: Future<Output = Result<Value>> + Send + 'static,
4136    {
4137        let state_factory = Arc::new(state_factory);
4138        let handler = Arc::new(handler);
4139
4140        let execute_factory = Arc::clone(&state_factory);
4141        let execute_handler = Arc::clone(&handler);
4142        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4143            let state = WorkflowInstance::new(execute_factory());
4144            let handler = Arc::clone(&execute_handler);
4145            Box::pin(async move {
4146                let result = handler(ctx, input.into_json()?, state).await?;
4147                AvroValue::from_serialize(&result)
4148            }) as WorkflowFuture
4149        });
4150
4151        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4152            let state = WorkflowInstance::new(state_factory());
4153            let snapshot_state = state.clone();
4154            let snapshot: WorkflowStateSnapshot =
4155                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4156            let replay_handler = Arc::clone(&handler);
4157            let future = async move {
4158                let result = replay_handler(ctx, input.into_json()?, state).await?;
4159                AvroValue::from_serialize(&result)
4160            };
4161            ReplayedWorkflowInvocation {
4162                future: Box::pin(future),
4163                snapshot,
4164            }
4165        });
4166
4167        self.workflows.insert(
4168            workflow_type.into(),
4169            RegisteredWorkflow {
4170                execute,
4171                replay: Some(replay),
4172                state_type: Some(TypeId::of::<S>()),
4173            },
4174        );
4175    }
4176
4177    /// Register a replayable workflow on the lossless fixed Avro Value surface.
4178    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4179        &mut self,
4180        workflow_type: impl Into<String>,
4181        state_factory: Factory,
4182        handler: F,
4183    ) where
4184        S: Clone + Send + Sync + 'static,
4185        Factory: Fn() -> S + Send + Sync + 'static,
4186        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4187        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4188    {
4189        let state_factory = Arc::new(state_factory);
4190        let handler = Arc::new(handler);
4191
4192        let execute_factory = Arc::clone(&state_factory);
4193        let execute_handler = Arc::clone(&handler);
4194        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4195            let state = WorkflowInstance::new(execute_factory());
4196            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4197        });
4198
4199        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4200            let state = WorkflowInstance::new(state_factory());
4201            let snapshot_state = state.clone();
4202            let snapshot: WorkflowStateSnapshot =
4203                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4204            ReplayedWorkflowInvocation {
4205                future: Box::pin(handler(ctx, input, state)),
4206                snapshot,
4207            }
4208        });
4209
4210        self.workflows.insert(
4211            workflow_type.into(),
4212            RegisteredWorkflow {
4213                execute,
4214                replay: Some(replay),
4215                state_type: Some(TypeId::of::<S>()),
4216            },
4217        );
4218    }
4219
4220    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4221    where
4222        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4223        Fut: Future<Output = Result<Value>> + Send + 'static,
4224    {
4225        let handler = Arc::new(handler);
4226        self.activities.insert(
4227            activity_type.into(),
4228            Arc::new(move |ctx, args| {
4229                let handler = Arc::clone(&handler);
4230                Box::pin(async move {
4231                    let result = handler(ctx, args.into_json()?).await?;
4232                    AvroValue::from_serialize(&result)
4233                })
4234            }),
4235        );
4236    }
4237
4238    /// Register an activity on the lossless fixed Avro Value surface.
4239    pub fn register_activity_avro_value<F, Fut>(
4240        &mut self,
4241        activity_type: impl Into<String>,
4242        handler: F,
4243    ) where
4244        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4245        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4246    {
4247        self.activities.insert(
4248            activity_type.into(),
4249            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4250        );
4251    }
4252
4253    /// Register a named, read-only query handler for a workflow type.
4254    ///
4255    /// The workflow type must also be registered with [`Worker::register_workflow`]
4256    /// before the worker runs. The handler receives only an immutable committed
4257    /// state snapshot and normalized query arguments.
4258    pub fn register_query<F, Fut>(
4259        &mut self,
4260        workflow_type: impl Into<String>,
4261        query_name: impl Into<String>,
4262        handler: F,
4263    ) where
4264        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4265        Fut: Future<Output = Result<Value>> + Send + 'static,
4266    {
4267        let handler = Arc::new(handler);
4268        self.queries
4269            .entry(workflow_type.into())
4270            .or_default()
4271            .insert(
4272                query_name.into(),
4273                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4274                    let handler = Arc::clone(&handler);
4275                    Box::pin(async move {
4276                        let result = handler(ctx, args.into_json()?).await?;
4277                        AvroValue::from_serialize(&result)
4278                    })
4279                })),
4280            );
4281    }
4282
4283    /// Register a query handler on the lossless fixed Avro Value surface.
4284    pub fn register_query_avro_value<F, Fut>(
4285        &mut self,
4286        workflow_type: impl Into<String>,
4287        query_name: impl Into<String>,
4288        handler: F,
4289    ) where
4290        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4291        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4292    {
4293        self.queries
4294            .entry(workflow_type.into())
4295            .or_default()
4296            .insert(
4297                query_name.into(),
4298                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4299            );
4300    }
4301
4302    /// Register a named query against deterministically replayed instance state.
4303    ///
4304    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
4305    /// same state type `S`. The handler receives an immutable, detached state
4306    /// clone, so successful and failed queries cannot affect workflow execution
4307    /// or the state reconstructed by a later query.
4308    pub fn register_replayed_query<S, F, Fut>(
4309        &mut self,
4310        workflow_type: impl Into<String>,
4311        query_name: impl Into<String>,
4312        handler: F,
4313    ) where
4314        S: Clone + Send + Sync + 'static,
4315        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4316        Fut: Future<Output = Result<Value>> + Send + 'static,
4317    {
4318        let handler = Arc::new(handler);
4319        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4320            let state = state.downcast::<S>().map_err(|_| {
4321                "registered query state type does not match the replayed workflow state".to_string()
4322            })?;
4323            let handler = Arc::clone(&handler);
4324            Ok(Box::pin(async move {
4325                let result = handler(ctx, state, args.into_json()?).await?;
4326                AvroValue::from_serialize(&result)
4327            }))
4328        });
4329
4330        self.queries
4331            .entry(workflow_type.into())
4332            .or_default()
4333            .insert(
4334                query_name.into(),
4335                RegisteredQuery::Replayed {
4336                    state_type: TypeId::of::<S>(),
4337                    handler: erased_handler,
4338                },
4339            );
4340    }
4341
4342    /// Register a replayed-state query on the lossless fixed Avro Value surface.
4343    pub fn register_replayed_query_avro_value<S, F, Fut>(
4344        &mut self,
4345        workflow_type: impl Into<String>,
4346        query_name: impl Into<String>,
4347        handler: F,
4348    ) where
4349        S: Clone + Send + Sync + 'static,
4350        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4351        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4352    {
4353        let handler = Arc::new(handler);
4354        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4355            let state = state.downcast::<S>().map_err(|_| {
4356                "registered query state type does not match the replayed workflow state".to_string()
4357            })?;
4358            Ok(Box::pin(handler(ctx, state, args)))
4359        });
4360
4361        self.queries
4362            .entry(workflow_type.into())
4363            .or_default()
4364            .insert(
4365                query_name.into(),
4366                RegisteredQuery::Replayed {
4367                    state_type: TypeId::of::<S>(),
4368                    handler: erased_handler,
4369                },
4370            );
4371    }
4372
4373    /// Register a synchronous workflow update handler.
4374    pub fn register_update<F, Fut>(
4375        &mut self,
4376        workflow_type: impl Into<String>,
4377        update_name: impl Into<String>,
4378        handler: F,
4379    ) where
4380        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4381        Fut: Future<Output = Result<Value>> + Send + 'static,
4382    {
4383        let handler = Arc::new(handler);
4384        self.updates
4385            .entry(workflow_type.into())
4386            .or_default()
4387            .insert(
4388                update_name.into(),
4389                Arc::new(move |ctx, args| {
4390                    let handler = Arc::clone(&handler);
4391                    Box::pin(async move {
4392                        let result = handler(ctx, args.into_json()?).await?;
4393                        AvroValue::from_serialize(&result)
4394                    })
4395                }),
4396            );
4397    }
4398
4399    /// Register an update handler on the lossless fixed Avro Value surface.
4400    pub fn register_update_avro_value<F, Fut>(
4401        &mut self,
4402        workflow_type: impl Into<String>,
4403        update_name: impl Into<String>,
4404        handler: F,
4405    ) where
4406        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4407        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4408    {
4409        self.updates
4410            .entry(workflow_type.into())
4411            .or_default()
4412            .insert(
4413                update_name.into(),
4414                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4415            );
4416    }
4417
4418    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4419        let mut command_contracts = serde_json::Map::new();
4420        for workflow_type in self.workflows.keys() {
4421            let mut queries = self
4422                .queries
4423                .get(workflow_type)
4424                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4425                .unwrap_or_default();
4426            queries.sort();
4427            let mut updates = self
4428                .updates
4429                .get(workflow_type)
4430                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4431                .unwrap_or_default();
4432            updates.sort();
4433            if !queries.is_empty() || !updates.is_empty() {
4434                command_contracts.insert(
4435                    workflow_type.clone(),
4436                    json!({
4437                        "queries": queries,
4438                        "updates": updates,
4439                    }),
4440                );
4441            }
4442        }
4443
4444        self.client
4445            .register_worker_with_command_contracts(
4446                &self.worker_id,
4447                &self.task_queue,
4448                self.workflows.keys().cloned().collect(),
4449                self.activities.keys().cloned().collect(),
4450                self.max_concurrent_workflow_tasks,
4451                self.max_concurrent_activity_tasks,
4452                [
4453                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4454                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4455                ]
4456                .into_iter()
4457                .flatten()
4458                .collect(),
4459                Value::Object(command_contracts),
4460            )
4461            .await
4462    }
4463
4464    /// Run until shutdown or a terminal worker error occurs.
4465    ///
4466    /// Empty long-poll expirations do not stop the worker. Retryable poll and
4467    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
4468    /// authentication, protocol, and other non-retryable failures are returned.
4469    pub async fn run(&self) -> Result<()> {
4470        self.run_until(std::future::pending::<()>()).await
4471    }
4472
4473    /// Run until `shutdown` resolves or a terminal worker error occurs.
4474    ///
4475    /// This has the same liveness and terminal-error contract as [`Worker::run`].
4476    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4477    where
4478        F: Future<Output = ()>,
4479    {
4480        let registration = self.register().await?;
4481        let heartbeat_interval = Duration::from_secs(
4482            registration
4483                .heartbeat_interval_seconds
4484                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4485        );
4486        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
4487        // from the completion of the preceding attempt, including its bounded
4488        // retries. A fixed-epoch interval can leave an already-due tick queued
4489        // while an acknowledgement is slow, producing a catch-up heartbeat as
4490        // soon as that request completes.
4491        let heartbeat = tokio::time::sleep(Duration::ZERO);
4492        tokio::pin!(heartbeat);
4493        tokio::pin!(shutdown);
4494        let stop = Arc::new(AtomicBool::new(false));
4495        // Poll responses may already have leased server-side work by the time
4496        // they become ready, so each poller owns its responses through
4497        // completion or failure instead of racing raw polls in this select.
4498        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4499            let worker = self.clone();
4500            let stop = Arc::clone(&stop);
4501            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4502        });
4503        let mut activity_poller = (!self.activities.is_empty()).then(|| {
4504            let worker = self.clone();
4505            let stop = Arc::clone(&stop);
4506            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4507        });
4508        let mut query_poller = (!self.queries.is_empty()).then(|| {
4509            let worker = self.clone();
4510            let stop = Arc::clone(&stop);
4511            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4512        });
4513
4514        loop {
4515            tokio::select! {
4516                _ = &mut shutdown => {
4517                    stop.store(true, Ordering::SeqCst);
4518                    break;
4519                }
4520                _ = &mut heartbeat => {
4521                    let result = self.retry_worker_operation(|| {
4522                        self.client.heartbeat_worker(
4523                            &self.worker_id,
4524                            self.max_concurrent_workflow_tasks,
4525                            self.max_concurrent_activity_tasks,
4526                        )
4527                    }).await;
4528                    heartbeat
4529                        .as_mut()
4530                        .reset(tokio::time::Instant::now() + heartbeat_interval);
4531                    match result {
4532                        Ok(acknowledgement) => {
4533                            if let Some(observer) = &self.heartbeat_observer {
4534                                observer(&WorkerHeartbeatObservation {
4535                                    worker_id: self.worker_id.clone(),
4536                                    task_queue: self.task_queue.clone(),
4537                                    acknowledged_at_unix_millis: SystemTime::now()
4538                                        .duration_since(UNIX_EPOCH)
4539                                        .unwrap_or_default()
4540                                        .as_millis()
4541                                        .min(u64::MAX as u128)
4542                                        as u64,
4543                                    acknowledgement,
4544                                });
4545                            }
4546                        }
4547                        Err(error) => {
4548                            stop.store(true, Ordering::SeqCst);
4549                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4550                            return Err(error);
4551                        }
4552                    }
4553                }
4554                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4555                    workflow_poller = None;
4556                    let stopped_by_server = stop.load(Ordering::SeqCst);
4557                    stop.store(true, Ordering::SeqCst);
4558                    let poller_result = optional_poller_result("workflow", result);
4559                    let join_result =
4560                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4561                    poller_result?;
4562                    join_result?;
4563                    if stopped_by_server {
4564                        return Ok(());
4565                    }
4566                    return Err(Error::WorkerLoop(
4567                        "workflow poller stopped unexpectedly".to_string(),
4568                    ));
4569                }
4570                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4571                    activity_poller = None;
4572                    let stopped_by_server = stop.load(Ordering::SeqCst);
4573                    stop.store(true, Ordering::SeqCst);
4574                    let poller_result = optional_poller_result("activity", result);
4575                    let join_result =
4576                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4577                    poller_result?;
4578                    join_result?;
4579                    if stopped_by_server {
4580                        return Ok(());
4581                    }
4582                    return Err(Error::WorkerLoop(
4583                        "activity poller stopped unexpectedly".to_string(),
4584                    ));
4585                }
4586                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4587                    query_poller = None;
4588                    let stopped_by_server = stop.load(Ordering::SeqCst);
4589                    stop.store(true, Ordering::SeqCst);
4590                    let poller_result = optional_poller_result("query", result);
4591                    let join_result =
4592                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4593                    poller_result?;
4594                    join_result?;
4595                    if stopped_by_server {
4596                        return Ok(());
4597                    }
4598                    return Err(Error::WorkerLoop(
4599                        "query poller stopped unexpectedly".to_string(),
4600                    ));
4601                }
4602            }
4603        }
4604
4605        join_pollers(
4606            workflow_poller.take(),
4607            activity_poller.take(),
4608            query_poller.take(),
4609        )
4610        .await
4611    }
4612
4613    /// Poll and settle at most one task from each enabled task family.
4614    ///
4615    /// A workflow may reach its server-enforced run deadline while this worker
4616    /// holds a task. When the completion endpoint authoritatively rejects that
4617    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
4618    /// terminal `run_status=failed`, the workflow tick is considered settled:
4619    /// the late command was not recorded and cannot replace the terminal run.
4620    /// Every other completion rejection remains an error. This worker-level
4621    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
4622    /// only bounds how long a client waits for a result.
4623    ///
4624    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
4625    /// the original [`Error::Http`] status and response body.
4626    pub async fn run_once(&self) -> Result<usize> {
4627        let mut handled = 0;
4628        match self.poll_workflow_once().await? {
4629            ManagedPollOutcome::Handled => handled += 1,
4630            ManagedPollOutcome::Stop => return Ok(handled),
4631            ManagedPollOutcome::Idle => {}
4632        }
4633        match self.poll_activity_once().await? {
4634            ManagedPollOutcome::Handled => handled += 1,
4635            ManagedPollOutcome::Stop => return Ok(handled),
4636            ManagedPollOutcome::Idle => {}
4637        }
4638        if !self.queries.is_empty() {
4639            match self.poll_query_once().await? {
4640                ManagedPollOutcome::Handled => handled += 1,
4641                ManagedPollOutcome::Stop => return Ok(handled),
4642                ManagedPollOutcome::Idle => {}
4643            }
4644        }
4645        Ok(handled)
4646    }
4647
4648    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4649        let poll_request_id = unique_request_id("rust-workflow-poll");
4650        let response = self
4651            .retry_worker_operation(|| {
4652                self.client.poll_workflow_task_response_with_request_id(
4653                    &self.worker_id,
4654                    &self.task_queue,
4655                    self.poll_timeout,
4656                    &poll_request_id,
4657                    0,
4658                )
4659            })
4660            .await?;
4661        if response.outcome().should_stop() {
4662            return Ok(ManagedPollOutcome::Stop);
4663        }
4664        let Some(task) = response.task else {
4665            return Ok(ManagedPollOutcome::Idle);
4666        };
4667
4668        let task_id = task.task_id.clone();
4669        let attempt = task.workflow_task_attempt;
4670        let run_id = task.run_id.clone();
4671        let lease_owner = task
4672            .lease_owner
4673            .clone()
4674            .unwrap_or_else(|| self.worker_id.clone());
4675
4676        match self.execute_workflow_task(task) {
4677            Ok(commands) if commands.is_empty() => {
4678                // A replay can consume a recorded pending durable command
4679                // without producing a new command. The standalone protocol
4680                // acknowledges that state through the typed waiting outcome;
4681                // an empty completion is rejected by servers that require at
4682                // least one executable command.
4683                self.client
4684                    .fail_workflow_task_with_type(
4685                        &task_id,
4686                        &lease_owner,
4687                        attempt,
4688                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4689                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4690                    )
4691                    .await?;
4692            }
4693            Ok(commands) => {
4694                let completion = self
4695                    .client
4696                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4697                    .await;
4698                if let Err(error) = completion {
4699                    if !workflow_task_completion_is_terminal_timeout(
4700                        &error,
4701                        &task_id,
4702                        attempt,
4703                        run_id.as_deref(),
4704                    ) {
4705                        return Err(error);
4706                    }
4707                }
4708            }
4709            Err(error) => {
4710                self.client
4711                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4712                    .await?;
4713            }
4714        }
4715
4716        Ok(ManagedPollOutcome::Handled)
4717    }
4718
4719    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4720        while !stop.load(Ordering::SeqCst) {
4721            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4722                stop.store(true, Ordering::SeqCst);
4723                break;
4724            }
4725        }
4726
4727        Ok(())
4728    }
4729
4730    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4731        let poll_request_id = unique_request_id("rust-activity-poll");
4732        let response = self
4733            .retry_worker_operation(|| {
4734                self.client.poll_activity_task_response_with_request_id(
4735                    &self.worker_id,
4736                    &self.task_queue,
4737                    self.poll_timeout,
4738                    &poll_request_id,
4739                    0,
4740                )
4741            })
4742            .await?;
4743        if response.outcome().should_stop() {
4744            return Ok(ManagedPollOutcome::Stop);
4745        }
4746        let Some(task) = response.task else {
4747            return Ok(ManagedPollOutcome::Idle);
4748        };
4749
4750        let task_id = task.task_id.clone();
4751        let attempt_id = task
4752            .activity_attempt_id
4753            .clone()
4754            .or(task.attempt_id.clone())
4755            .unwrap_or_default();
4756        let lease_owner = task
4757            .lease_owner
4758            .clone()
4759            .unwrap_or_else(|| self.worker_id.clone());
4760        let codec = task.payload_codec.clone();
4761        let result = self.execute_activity_task(task).await;
4762        match result {
4763            Ok(value) => {
4764                let completion = self
4765                    .client
4766                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4767                    .await;
4768                if let Err(error) = completion {
4769                    if !activity_task_rejection_is_final(&error) {
4770                        return Err(error);
4771                    }
4772                }
4773            }
4774            Err(error) => {
4775                let failure = self
4776                    .client
4777                    .fail_activity_task(
4778                        &task_id,
4779                        &attempt_id,
4780                        &lease_owner,
4781                        error.to_string(),
4782                        false,
4783                    )
4784                    .await;
4785                if let Err(error) = failure {
4786                    if !activity_task_rejection_is_final(&error) {
4787                        return Err(error);
4788                    }
4789                }
4790            }
4791        }
4792
4793        Ok(ManagedPollOutcome::Handled)
4794    }
4795
4796    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4797        while !stop.load(Ordering::SeqCst) {
4798            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4799                stop.store(true, Ordering::SeqCst);
4800                break;
4801            }
4802        }
4803
4804        Ok(())
4805    }
4806
4807    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4808        let poll_request_id = unique_request_id("rust-query-poll");
4809        let response = self
4810            .retry_worker_operation(|| {
4811                self.client.poll_query_task_response_with_request_id(
4812                    &self.worker_id,
4813                    &self.task_queue,
4814                    self.poll_timeout,
4815                    &poll_request_id,
4816                    0,
4817                )
4818            })
4819            .await?;
4820        if response.outcome().should_stop() {
4821            return Ok(ManagedPollOutcome::Stop);
4822        }
4823        let Some(task) = response.task else {
4824            return Ok(ManagedPollOutcome::Idle);
4825        };
4826
4827        let query_task_id = task.query_task_id.clone();
4828        let attempt = task.query_task_attempt;
4829        let lease_owner = task
4830            .lease_owner
4831            .clone()
4832            .unwrap_or_else(|| self.worker_id.clone());
4833        let codec = task.payload_codec.clone();
4834
4835        match self.execute_query_task(task).await {
4836            Ok(value) => {
4837                let result_envelope = match encode_typed_envelope(&value, &codec) {
4838                    Ok(result_envelope) => result_envelope,
4839                    Err(error) => {
4840                        let failure = self
4841                            .client
4842                            .fail_query_task(
4843                                &query_task_id,
4844                                &lease_owner,
4845                                attempt,
4846                                error.to_string(),
4847                                "query_result_encode_failed",
4848                                "QueryResultEncodeFailed",
4849                            )
4850                            .await;
4851                        if let Err(error) = failure {
4852                            if !query_task_rejection_is_final(&error) {
4853                                return Err(error);
4854                            }
4855                        }
4856                        return Ok(ManagedPollOutcome::Handled);
4857                    }
4858                };
4859
4860                if let Err(error) = self
4861                    .client
4862                    .complete_query_task_with_envelope(
4863                        &query_task_id,
4864                        &lease_owner,
4865                        attempt,
4866                        value.clone().into_json()?,
4867                        result_envelope,
4868                    )
4869                    .await
4870                {
4871                    if !query_task_rejection_is_final(&error) {
4872                        return Err(error);
4873                    }
4874                }
4875            }
4876            Err(failure) => {
4877                let result = self
4878                    .client
4879                    .fail_query_task(
4880                        &query_task_id,
4881                        &lease_owner,
4882                        attempt,
4883                        failure.message,
4884                        failure.reason,
4885                        failure.failure_type,
4886                    )
4887                    .await;
4888                if let Err(error) = result {
4889                    if !query_task_rejection_is_final(&error) {
4890                        return Err(error);
4891                    }
4892                }
4893            }
4894        }
4895
4896        Ok(ManagedPollOutcome::Handled)
4897    }
4898
4899    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4900        while !stop.load(Ordering::SeqCst) {
4901            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
4902                stop.store(true, Ordering::SeqCst);
4903                break;
4904            }
4905        }
4906
4907        Ok(())
4908    }
4909
4910    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
4911    where
4912        F: FnMut() -> Fut,
4913        Fut: Future<Output = Result<T>>,
4914    {
4915        let mut retries = 0;
4916
4917        loop {
4918            match operation().await {
4919                Err(error)
4920                    if worker_operation_is_retryable(&error)
4921                        && retries < self.retry_policy.max_retries =>
4922                {
4923                    retries += 1;
4924                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
4925                }
4926                result => return result,
4927            }
4928        }
4929    }
4930
4931    async fn execute_query_task(
4932        &self,
4933        mut task: QueryTask,
4934    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
4935        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
4936            return Err(QueryTaskExecutionFailure::new(
4937                "query_payload_decode_failed",
4938                format!(
4939                    "cannot decode query payload with unsupported codec {:?}",
4940                    task.payload_codec
4941                ),
4942                "QueryPayloadDecodeFailed",
4943            ));
4944        }
4945
4946        if !self.workflows.contains_key(&task.workflow_type) {
4947            return Err(QueryTaskExecutionFailure::new(
4948                "query_workflow_type_not_registered",
4949                format!("no workflow registered for type {:?}", task.workflow_type),
4950                "WorkflowTypeNotRegistered",
4951            ));
4952        }
4953
4954        let Some(handlers) = self.queries.get(&task.workflow_type) else {
4955            return Err(QueryTaskExecutionFailure::new(
4956                "query_handler_unavailable",
4957                format!(
4958                    "query handlers are unavailable for workflow type {:?}",
4959                    task.workflow_type
4960                ),
4961                "QueryHandlerUnavailable",
4962            ));
4963        };
4964        let Some(query) = handlers.get(&task.query_name) else {
4965            return Err(QueryTaskExecutionFailure::new(
4966                "rejected_unknown_query",
4967                format!("unknown query {:?}", task.query_name),
4968                "QueryFailed",
4969            ));
4970        };
4971
4972        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
4973            .map_err(|error| {
4974            QueryTaskExecutionFailure::new(
4975                "query_payload_decode_failed",
4976                format!("cannot decode query arguments: {error}"),
4977                "QueryPayloadDecodeFailed",
4978            )
4979        })?;
4980        let workflow_input_typed =
4981            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
4982                .map_err(|error| {
4983                    QueryTaskExecutionFailure::new(
4984                        "query_workflow_state_unavailable",
4985                        format!("cannot decode workflow start input: {error}"),
4986                        "QueryWorkflowStateUnavailable",
4987                    )
4988                })?;
4989        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
4990            QueryTaskExecutionFailure::new(
4991                "query_workflow_state_unavailable",
4992                format!("cannot project workflow start input: {error}"),
4993                "QueryWorkflowStateUnavailable",
4994            )
4995        })?;
4996        hydrate_query_history_from_export(&mut task).map_err(|error| {
4997            QueryTaskExecutionFailure::new(
4998                "query_workflow_state_unavailable",
4999                format!("cannot restore query history snapshot: {error}"),
5000                "QueryWorkflowStateUnavailable",
5001            )
5002        })?;
5003        enrich_query_history_from_export(&mut task).map_err(|error| {
5004            QueryTaskExecutionFailure::new(
5005                "query_workflow_state_unavailable",
5006                format!("cannot restore compact query history payloads: {error}"),
5007                "QueryWorkflowStateUnavailable",
5008            )
5009        })?;
5010        let signal_events = query_signal_events(&task).map_err(|error| {
5011            QueryTaskExecutionFailure::new(
5012                "query_workflow_state_unavailable",
5013                format!("cannot decode committed workflow signals: {error}"),
5014                "QueryWorkflowStateUnavailable",
5015            )
5016        })?;
5017        let history_events = Arc::new(std::mem::take(&mut task.history_events));
5018        let context = QueryContext {
5019            workflow_id: task.workflow_id,
5020            run_id: task.run_id,
5021            workflow_type: task.workflow_type.clone(),
5022            run_status: task.run_status,
5023            workflow_input,
5024            workflow_input_avro_value: workflow_input_typed.clone(),
5025            history_events: Arc::clone(&history_events),
5026            signal_events: Arc::new(signal_events),
5027        };
5028
5029        let future = match query {
5030            RegisteredQuery::Snapshot(handler) => handler(context, args),
5031            RegisteredQuery::Replayed {
5032                state_type,
5033                handler,
5034            } => {
5035                let workflow = self
5036                    .workflows
5037                    .get(&task.workflow_type)
5038                    .expect("workflow registration was checked above");
5039                if workflow.state_type != Some(*state_type) {
5040                    return Err(QueryTaskExecutionFailure::new(
5041                        "query_workflow_state_unavailable",
5042                        "replayed query state type does not match its workflow registration",
5043                        "QueryWorkflowStateUnavailable",
5044                    ));
5045                }
5046                let replay = workflow.replay.as_ref().ok_or_else(|| {
5047                    QueryTaskExecutionFailure::new(
5048                        "query_workflow_state_unavailable",
5049                        format!(
5050                            "workflow type {:?} is not registered for instance-state replay",
5051                            task.workflow_type
5052                        ),
5053                        "QueryWorkflowStateUnavailable",
5054                    )
5055                })?;
5056                let workflow_state = Arc::new(Mutex::new(
5057                    WorkflowState::new_with_identity(
5058                        history_events.as_ref().clone(),
5059                        context.workflow_id.clone(),
5060                        context.run_id.clone(),
5061                        self.task_queue.clone(),
5062                        task.payload_codec,
5063                        None,
5064                    )
5065                    .map_err(|error| {
5066                        QueryTaskExecutionFailure::new(
5067                            "query_workflow_state_unavailable",
5068                            format!("workflow replay failed before query: {error}"),
5069                            "QueryWorkflowStateUnavailable",
5070                        )
5071                    })?,
5072                ));
5073                let workflow_context = WorkflowContext {
5074                    state: workflow_state,
5075                };
5076                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5077                let mut cx = TaskContext::from_waker(noop_waker_ref());
5078                match invocation.future.as_mut().poll(&mut cx) {
5079                    Poll::Ready(Ok(_)) => {
5080                        workflow_context
5081                            .ensure_history_consumed()
5082                            .map_err(|error| {
5083                                QueryTaskExecutionFailure::new(
5084                                    "query_workflow_state_unavailable",
5085                                    format!("workflow replay failed before query: {error}"),
5086                                    "QueryWorkflowStateUnavailable",
5087                                )
5088                            })?;
5089                    }
5090                    Poll::Ready(Err(error)) => {
5091                        return Err(QueryTaskExecutionFailure::new(
5092                            "query_workflow_state_unavailable",
5093                            format!("workflow replay failed before query: {error}"),
5094                            "QueryWorkflowStateUnavailable",
5095                        ));
5096                    }
5097                    Poll::Pending => {
5098                        let commands = workflow_context.take_commands().map_err(|error| {
5099                            QueryTaskExecutionFailure::new(
5100                                "query_workflow_state_unavailable",
5101                                format!("workflow replay failed before query: {error}"),
5102                                "QueryWorkflowStateUnavailable",
5103                            )
5104                        })?;
5105                        if commands.is_empty()
5106                            && !workflow_context
5107                                .matched_recorded_pending()
5108                                .map_err(|error| {
5109                                    QueryTaskExecutionFailure::new(
5110                                        "query_workflow_state_unavailable",
5111                                        format!("workflow replay failed before query: {error}"),
5112                                        "QueryWorkflowStateUnavailable",
5113                                    )
5114                                })?
5115                        {
5116                            return Err(QueryTaskExecutionFailure::new(
5117                                "query_workflow_state_unavailable",
5118                                "workflow replay yielded without a durable command",
5119                                "QueryWorkflowStateUnavailable",
5120                            ));
5121                        }
5122                    }
5123                }
5124                let state = (invocation.snapshot)().map_err(|error| {
5125                    QueryTaskExecutionFailure::new(
5126                        "query_workflow_state_unavailable",
5127                        format!("cannot snapshot replayed workflow state: {error}"),
5128                        "QueryWorkflowStateUnavailable",
5129                    )
5130                })?;
5131                handler(context, state, args).map_err(|message| {
5132                    QueryTaskExecutionFailure::new(
5133                        "query_workflow_state_unavailable",
5134                        message,
5135                        "QueryWorkflowStateUnavailable",
5136                    )
5137                })?
5138            }
5139        };
5140
5141        future.await.map_err(|error| {
5142            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5143        })
5144    }
5145
5146    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5147        if let Some(update_id) = task
5148            .workflow_update_id
5149            .as_deref()
5150            .filter(|update_id| !update_id.is_empty())
5151        {
5152            return self.execute_update_task(&task, update_id);
5153        }
5154
5155        let workflow = self
5156            .workflows
5157            .get(&task.workflow_type)
5158            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5159        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5160        let resume_signal = decode_resume_signal(&task)?;
5161        let history_budget = WorkflowHistoryBudget {
5162            event_count: task
5163                .total_history_events
5164                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5165            size_bytes: task.history_size_bytes,
5166            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5167            pressure: task.history_budget_pressure.clone(),
5168        };
5169        let mut workflow_state = WorkflowState::new_with_identity(
5170            task.history_events,
5171            task.workflow_id,
5172            task.run_id,
5173            self.task_queue.clone(),
5174            task.payload_codec.clone(),
5175            resume_signal,
5176        )?;
5177        workflow_state.history_budget = history_budget;
5178        let state = Arc::new(Mutex::new(workflow_state));
5179        let ctx = WorkflowContext { state };
5180        let mut future = (workflow.execute)(ctx.clone(), input);
5181        let mut cx = TaskContext::from_waker(noop_waker_ref());
5182
5183        match future.as_mut().poll(&mut cx) {
5184            Poll::Ready(Ok(result)) => {
5185                ctx.ensure_history_consumed()?;
5186                let result = encode_typed_envelope(&result, &task.payload_codec)?;
5187                let mut commands = ctx.take_commands()?;
5188                commands.push(json!({
5189                    "type": "complete_workflow",
5190                    "result": result
5191                }));
5192                Ok(commands)
5193            }
5194            Poll::Ready(Err(error)) => {
5195                if let Error::ContinueAsNew(request) = error {
5196                    let mut commands = ctx.take_commands()?;
5197                    if let Some(command) = ctx.continue_as_new_command(request)? {
5198                        commands.push(command);
5199                    }
5200                    ctx.ensure_history_consumed()?;
5201                    return Ok(commands);
5202                }
5203                // A handler error must not hide a committed durable command that
5204                // upgraded workflow code no longer consumes.
5205                ctx.ensure_history_consumed()?;
5206                if workflow_task_integrity_error(&error) {
5207                    // Replay and protocol failures describe the workflow-task
5208                    // decision itself. Do not let commands queued earlier in
5209                    // this uncommitted decision escape alongside a terminal
5210                    // workflow failure.
5211                    return Err(error);
5212                }
5213                let mut commands = ctx.take_commands()?;
5214                commands.push(workflow_failure_command(&error));
5215                Ok(commands)
5216            }
5217            Poll::Pending => {
5218                let commands = ctx.take_commands()?;
5219                if commands.is_empty() && !ctx.matched_recorded_pending()? {
5220                    Err(Error::WorkflowYieldedWithoutCommand)
5221                } else {
5222                    Ok(commands)
5223                }
5224            }
5225        }
5226    }
5227
5228    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5229        if !self.workflows.contains_key(&task.workflow_type) {
5230            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5231        }
5232
5233        let accepted = task.history_events.iter().rev().find_map(|event| {
5234            (event.event_type == "UpdateAccepted"
5235                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5236            .then_some(&event.payload)
5237        });
5238        let update_name = accepted
5239            .and_then(|payload| payload.get("update_name"))
5240            .and_then(Value::as_str)
5241            .or(task.update_name.as_deref())
5242            .unwrap_or_default();
5243        let Some(handler) = self
5244            .updates
5245            .get(&task.workflow_type)
5246            .and_then(|handlers| handlers.get(update_name))
5247        else {
5248            return Ok(vec![json!({
5249                "type": "fail_update",
5250                "update_id": update_id,
5251                "message": format!(
5252                    "no update handler is registered for {}.{update_name}",
5253                    task.workflow_type
5254                ),
5255                "exception_type": "UnknownUpdate",
5256                "non_retryable": true,
5257            })]);
5258        };
5259        let arguments = accepted
5260            .and_then(|payload| payload.get("arguments"))
5261            .or(task.arguments.as_ref());
5262        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5263        let context = QueryContext {
5264            workflow_id: task.workflow_id.clone(),
5265            run_id: task.run_id.clone(),
5266            workflow_type: task.workflow_type.clone(),
5267            run_status: Some("running".to_string()),
5268            workflow_input: Value::Null,
5269            workflow_input_avro_value: AvroValue::Null,
5270            history_events: Arc::new(task.history_events.clone()),
5271            signal_events: Arc::new(Vec::new()),
5272        };
5273        let mut future = handler(context, arguments);
5274        let mut cx = TaskContext::from_waker(noop_waker_ref());
5275
5276        match future.as_mut().poll(&mut cx) {
5277            Poll::Ready(Ok(result)) => Ok(vec![json!({
5278                "type": "complete_update",
5279                "update_id": update_id,
5280                "result": encode_typed_envelope(&result, &task.payload_codec)?,
5281            })]),
5282            Poll::Ready(Err(error)) => Ok(vec![json!({
5283                "type": "fail_update",
5284                "update_id": update_id,
5285                "message": error.to_string(),
5286                "exception_type": "UpdateFailed",
5287                "non_retryable": true,
5288            })]),
5289            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5290        }
5291    }
5292
5293    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5294        let handler = self
5295            .activities
5296            .get(&task.activity_type)
5297            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5298        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5299        let attempt_id = task
5300            .activity_attempt_id
5301            .clone()
5302            .or(task.attempt_id.clone())
5303            .unwrap_or_default();
5304        let lease_owner = task
5305            .lease_owner
5306            .clone()
5307            .unwrap_or_else(|| self.worker_id.clone());
5308        let ctx = ActivityContext {
5309            client: self.client.clone(),
5310            task_id: task.task_id,
5311            activity_attempt_id: attempt_id,
5312            lease_owner,
5313            activity_type: task.activity_type,
5314            attempt_number: task.attempt_number,
5315            task_queue: self.task_queue.clone(),
5316            worker_id: self.worker_id.clone(),
5317        };
5318
5319        handler(ctx, args).await
5320    }
5321}
5322
5323fn poller_result(
5324    kind: &str,
5325    result: std::result::Result<Result<()>, tokio::task::JoinError>,
5326) -> Result<()> {
5327    match result {
5328        Ok(result) => result,
5329        Err(error) => Err(Error::WorkerLoop(format!(
5330            "{kind} poller join error: {error}"
5331        ))),
5332    }
5333}
5334
5335fn optional_poller_result(
5336    kind: &str,
5337    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5338) -> Result<()> {
5339    match result {
5340        Some(result) => poller_result(kind, result),
5341        None => Ok(()),
5342    }
5343}
5344
5345async fn join_pollers(
5346    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5347    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5348    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5349) -> Result<()> {
5350    let mut first_error = None;
5351
5352    if let Some(handle) = workflow_poller {
5353        if let Err(error) = poller_result("workflow", handle.await) {
5354            first_error.get_or_insert(error);
5355        }
5356    }
5357
5358    if let Some(handle) = activity_poller {
5359        if let Err(error) = poller_result("activity", handle.await) {
5360            first_error.get_or_insert(error);
5361        }
5362    }
5363
5364    if let Some(handle) = query_poller {
5365        if let Err(error) = poller_result("query", handle.await) {
5366            first_error.get_or_insert(error);
5367        }
5368    }
5369
5370    if let Some(error) = first_error {
5371        Err(error)
5372    } else {
5373        Ok(())
5374    }
5375}
5376
5377fn default_worker_id() -> String {
5378    let millis = SystemTime::now()
5379        .duration_since(UNIX_EPOCH)
5380        .unwrap_or_default()
5381        .as_millis();
5382    format!("rust-worker-{}-{millis}", std::process::id())
5383}
5384
5385fn unique_request_id(prefix: &str) -> String {
5386    let nanos = SystemTime::now()
5387        .duration_since(UNIX_EPOCH)
5388        .unwrap_or_default()
5389        .as_nanos();
5390    format!("{prefix}-{}-{nanos}", std::process::id())
5391}
5392
5393#[derive(Debug)]
5394struct QueryTaskExecutionFailure {
5395    reason: String,
5396    message: String,
5397    failure_type: String,
5398}
5399
5400impl QueryTaskExecutionFailure {
5401    fn new(
5402        reason: impl Into<String>,
5403        message: impl Into<String>,
5404        failure_type: impl Into<String>,
5405    ) -> Self {
5406        Self {
5407            reason: reason.into(),
5408            message: message.into(),
5409            failure_type: failure_type.into(),
5410        }
5411    }
5412}
5413
5414/// Typed local state owned by one deterministic workflow invocation.
5415///
5416/// Use [`WorkflowInstance::update`] for the same state transitions during
5417/// ordinary execution and replay. A replayed query receives a detached
5418/// immutable `Arc<S>` rather than this mutation-capable handle.
5419#[derive(Clone, Debug)]
5420pub struct WorkflowInstance<S> {
5421    state: Arc<Mutex<S>>,
5422}
5423
5424impl<S> WorkflowInstance<S> {
5425    fn new(state: S) -> Self {
5426        Self {
5427            state: Arc::new(Mutex::new(state)),
5428        }
5429    }
5430
5431    /// Read the current workflow-instance state without changing it.
5432    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5433        let state = self
5434            .state
5435            .lock()
5436            .map_err(|_| Error::WorkflowStatePoisoned)?;
5437        Ok(reader(&state))
5438    }
5439
5440    /// Apply one deterministic workflow-instance state transition.
5441    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5442        let mut state = self
5443            .state
5444            .lock()
5445            .map_err(|_| Error::WorkflowStatePoisoned)?;
5446        Ok(transition(&mut state))
5447    }
5448}
5449
5450impl<S: Clone> WorkflowInstance<S> {
5451    fn snapshot(&self) -> Result<S> {
5452        self.read(Clone::clone)
5453    }
5454}
5455
5456#[derive(Clone, Debug)]
5457pub struct WorkflowContext {
5458    state: Arc<Mutex<WorkflowState>>,
5459}
5460
5461impl WorkflowContext {
5462    /// Identity of the parent workflow currently being replayed.
5463    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5464        let state = self
5465            .state
5466            .lock()
5467            .map_err(|_| Error::WorkflowStatePoisoned)?;
5468        Ok(WorkflowIdentity {
5469            workflow_id: state.workflow_id.clone(),
5470            run_id: state.run_id.clone(),
5471        })
5472    }
5473
5474    /// Return the server-published history budget for this workflow task.
5475    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5476        let state = self
5477            .state
5478            .lock()
5479            .map_err(|_| Error::WorkflowStatePoisoned)?;
5480        Ok(state.history_budget.clone())
5481    }
5482
5483    /// Continue this workflow instance as a fresh run with replacement arguments.
5484    ///
5485    /// Return this value directly from the workflow handler. The worker converts
5486    /// it to the terminal protocol command only after replay has consumed every
5487    /// recorded durable command.
5488    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5489        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5490    }
5491
5492    /// Continue as new with optional workflow-type and task-queue overrides.
5493    pub fn continue_as_new_with_options<T: Serialize>(
5494        &self,
5495        options: ContinueAsNewOptions,
5496        args: T,
5497    ) -> Result<Value> {
5498        options.validate()?;
5499        Err(Error::ContinueAsNew(ContinueAsNewRequest {
5500            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5501            options,
5502        }))
5503    }
5504
5505    pub fn activity<T: Serialize>(
5506        &self,
5507        activity_type: impl Into<String>,
5508        args: T,
5509    ) -> ActivityCall {
5510        self.activity_with_options(activity_type, ActivityOptions::new(), args)
5511    }
5512
5513    pub fn activity_on_queue<T, Q>(
5514        &self,
5515        activity_type: impl Into<String>,
5516        task_queue: Option<Q>,
5517        args: T,
5518    ) -> ActivityCall
5519    where
5520        T: Serialize,
5521        Q: Into<String>,
5522    {
5523        let mut options = ActivityOptions::new();
5524        options.task_queue = task_queue.map(Into::into);
5525        self.activity_with_options(activity_type, options, args)
5526    }
5527
5528    /// Schedule one durable activity with retry, routing, and timeout options.
5529    ///
5530    /// Options are validated before the command is emitted. Once the command is
5531    /// recorded, replay consumes the same activity lifecycle at this command
5532    /// position and never emits a duplicate schedule.
5533    ///
5534    /// ```no_run
5535    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
5536    /// # use std::time::Duration;
5537    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
5538    /// let result = ctx
5539    ///     .activity_with_options(
5540    ///         "charge-card",
5541    ///         ActivityOptions::new()
5542    ///             .task_queue("payments")
5543    ///             .retry_policy(
5544    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
5545    ///                     Duration::from_secs(1),
5546    ///                     2,
5547    ///                     Some(Duration::from_secs(30)),
5548    ///                 ),
5549    ///             )
5550    ///             .start_to_close_timeout(Duration::from_secs(60))
5551    ///             .schedule_to_close_timeout(Duration::from_secs(180))
5552    ///             .heartbeat_timeout(Duration::from_secs(15)),
5553    ///         json!([{"order_id": "order-42"}]),
5554    ///     )
5555    ///     .await;
5556    /// match result {
5557    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
5558    ///         "reason": failure.reason,
5559    ///         "timeout_kind": failure.timeout_kind,
5560    ///     })),
5561    ///     other => other,
5562    /// }
5563    /// # }
5564    /// ```
5565    pub fn activity_with_options<T: Serialize>(
5566        &self,
5567        activity_type: impl Into<String>,
5568        options: ActivityOptions,
5569        args: T,
5570    ) -> ActivityCall {
5571        ActivityCall {
5572            ctx: self.clone(),
5573            activity_type: activity_type.into(),
5574            options,
5575            args: Some(AvroValue::from_serialize(&args)),
5576            scheduled: false,
5577        }
5578    }
5579
5580    pub async fn activity_avro_value<T: Serialize>(
5581        &self,
5582        activity_type: impl Into<String>,
5583        args: T,
5584    ) -> Result<AvroValue> {
5585        let mut call = self.activity(activity_type, args);
5586        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5587    }
5588
5589    pub async fn activity_avro_value_with_options<T: Serialize>(
5590        &self,
5591        activity_type: impl Into<String>,
5592        options: ActivityOptions,
5593        args: T,
5594    ) -> Result<AvroValue> {
5595        let mut call = self.activity_with_options(activity_type, options, args);
5596        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5597    }
5598
5599    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5600        SignalCall {
5601            ctx: self.clone(),
5602            signal_name: signal_name.into(),
5603            opened_wait: false,
5604            matched_pending: false,
5605        }
5606    }
5607
5608    pub async fn wait_signal_avro_value(
5609        &self,
5610        signal_name: impl Into<String>,
5611    ) -> Result<Vec<AvroValue>> {
5612        let mut call = self.wait_signal(signal_name);
5613        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5614    }
5615
5616    /// Wait for server-backed durable time without blocking the worker executor.
5617    ///
5618    /// Polling this future emits one `start_timer` command and yields. The
5619    /// server records the deadline, so neither worker nor server restarts reset
5620    /// the wait. Replay resolves the future only from a `TimerScheduled` and
5621    /// `TimerFired` pair at the same position in the shared durable-command
5622    /// stream, with matching sequence, timer identity, and delay. Sub-second
5623    /// durations round up because protocol deadlines use whole seconds.
5624    ///
5625    /// ```no_run
5626    /// # use durable_workflow::{json, Client, Worker};
5627    /// # use std::time::Duration;
5628    /// # fn configure(client: Client) {
5629    /// let mut worker = Worker::new(client, "rust-workers");
5630    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
5631    ///     ctx.sleep(Duration::from_secs(5)).await?;
5632    ///     Ok(json!({"status": "timer fired"}))
5633    /// });
5634    /// # }
5635    /// ```
5636    pub fn sleep(&self, duration: Duration) -> TimerCall {
5637        let delay_seconds = duration
5638            .as_secs()
5639            .checked_add(u64::from(duration.subsec_nanos() > 0));
5640        TimerCall {
5641            ctx: self.clone(),
5642            delay_seconds,
5643            scheduled: false,
5644            matched_pending: false,
5645        }
5646    }
5647
5648    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
5649    pub fn start_timer(&self, duration: Duration) -> TimerCall {
5650        self.sleep(duration)
5651    }
5652
5653    /// Evaluate a non-deterministic callback once and durably record its typed value.
5654    ///
5655    /// On replay the callback is not invoked: the value is decoded from the
5656    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
5657    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
5658    /// small values that must remain fixed for the lifetime of a workflow run.
5659    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5660    where
5661        T: Serialize + DeserializeOwned,
5662        F: FnOnce() -> T,
5663    {
5664        {
5665            let mut state = self
5666                .state
5667                .lock()
5668                .map_err(|_| Error::WorkflowStatePoisoned)?;
5669            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5670                return match recorded {
5671                    RecordedCommand::SideEffect { sequence, value } => {
5672                        state.command_cursor += 1;
5673                        value.deserialize().map_err(|error| {
5674                            Error::NonDeterministicReplay(ReplayFailure::new(
5675                                "side_effect_type_mismatch",
5676                                Some(sequence),
5677                                Some(std::any::type_name::<T>().to_string()),
5678                                Some(error.to_string()),
5679                                "recorded side-effect value is incompatible with the requested Rust type",
5680                            ))
5681                        })
5682                    }
5683                    other => Err(command_mismatch(&other, "side effect")),
5684                };
5685            }
5686        }
5687
5688        let value = callback();
5689        let avro_value = AvroValue::from_serialize(&value)?;
5690        let mut state = self
5691            .state
5692            .lock()
5693            .map_err(|_| Error::WorkflowStatePoisoned)?;
5694        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5695        state.commands.push(json!({
5696            "type": "record_side_effect",
5697            "result": result,
5698        }));
5699        Ok(value)
5700    }
5701
5702    /// Record or replay a lossless fixed Avro Value side effect.
5703    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5704    where
5705        F: FnOnce() -> AvroValue,
5706    {
5707        {
5708            let mut state = self
5709                .state
5710                .lock()
5711                .map_err(|_| Error::WorkflowStatePoisoned)?;
5712            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5713                return match recorded {
5714                    RecordedCommand::SideEffect { value, .. } => {
5715                        state.command_cursor += 1;
5716                        Ok(value)
5717                    }
5718                    other => Err(command_mismatch(&other, "side effect")),
5719                };
5720            }
5721        }
5722
5723        let value = callback();
5724        let mut state = self
5725            .state
5726            .lock()
5727            .map_err(|_| Error::WorkflowStatePoisoned)?;
5728        let result = encode_typed_envelope(&value, &state.payload_codec)?;
5729        state.commands.push(json!({
5730            "type": "record_side_effect",
5731            "result": result,
5732        }));
5733        Ok(value)
5734    }
5735
5736    /// Record a UUIDv4 once and return the same UUID on every replay.
5737    pub fn uuid_v4(&self) -> Result<Uuid> {
5738        self.side_effect(Uuid::new_v4)
5739    }
5740
5741    /// Select the newest supported version for a change, or replay the version
5742    /// already committed for that stable change ID.
5743    pub fn get_version(
5744        &self,
5745        change_id: impl Into<String>,
5746        min_supported: i32,
5747        max_supported: i32,
5748    ) -> Result<i32> {
5749        let change_id = change_id.into();
5750        if change_id.trim().is_empty() {
5751            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5752                "version_change_id_invalid",
5753                None,
5754                Some("non-empty change ID".to_string()),
5755                Some(change_id),
5756                "version markers require a stable non-empty change ID",
5757            )));
5758        }
5759        if min_supported > max_supported {
5760            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5761                "version_range_invalid",
5762                None,
5763                Some("min_supported <= max_supported".to_string()),
5764                Some(format!("{min_supported}..={max_supported}")),
5765                "version marker supported range is invalid",
5766            )));
5767        }
5768
5769        let mut state = self
5770            .state
5771            .lock()
5772            .map_err(|_| Error::WorkflowStatePoisoned)?;
5773        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5774            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5775            return Ok(version);
5776        }
5777
5778        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5779            return match recorded {
5780                RecordedCommand::VersionMarker {
5781                    sequence,
5782                    change_id: recorded_change_id,
5783                    version,
5784                    ..
5785                } => {
5786                    if recorded_change_id != change_id {
5787                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5788                            "version_change_id_mismatch",
5789                            Some(sequence),
5790                            Some(recorded_change_id),
5791                            Some(change_id),
5792                            "recorded version marker change ID differs from current workflow code",
5793                        )));
5794                    }
5795                    ensure_version_supported(
5796                        &change_id,
5797                        version,
5798                        min_supported,
5799                        max_supported,
5800                        sequence,
5801                    )?;
5802                    state.command_cursor += 1;
5803                    state.version_markers.insert(change_id, (version, sequence));
5804                    Ok(version)
5805                }
5806                other => Err(command_mismatch(
5807                    &other,
5808                    format!("version marker:{change_id}"),
5809                )),
5810            };
5811        }
5812
5813        let version = max_supported;
5814        state.commands.push(json!({
5815            "type": "record_version_marker",
5816            "change_id": change_id,
5817            "version": version,
5818            "min_supported": min_supported,
5819            "max_supported": max_supported,
5820        }));
5821        // Sequence numbers are assigned by the server. Zero identifies a marker
5822        // selected in this uncommitted decision batch for duplicate-call checks.
5823        state.version_markers.insert(change_id, (version, 0));
5824        Ok(version)
5825    }
5826
5827    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
5828    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5829        Ok(self.get_version(change_id, -1, 1)? == 1)
5830    }
5831
5832    /// Keep a patch marker in history after the legacy branch has been removed.
5833    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5834        self.get_version(change_id, -1, 1).map(|_| ())
5835    }
5836
5837    /// Start a named durable child on an explicit queue and await its result.
5838    ///
5839    /// The command is recorded in the parent's sequence-ordered durable command
5840    /// stream. Replay keeps a scheduled child pending without emitting another
5841    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
5842    /// values preserve the history payload codec and include both sides of the
5843    /// durable relationship; failures are returned as
5844    /// [`Error::ChildWorkflowFailed`].
5845    ///
5846    /// ```no_run
5847    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
5848    /// # fn configure(client: Client) {
5849    /// let mut worker = Worker::new(client, "parent-workers");
5850    /// worker.register_workflow("order-parent", |ctx, _input| async move {
5851    ///     let child = ctx
5852    ///         .start_child_workflow(
5853    ///             "fulfil-order",
5854    ///             ChildWorkflowOptions::new("fulfilment-workers")
5855    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
5856    ///             json!([{"order_id": "order-42"}]),
5857    ///         )
5858    ///         .await?;
5859    ///     Ok(child.result)
5860    /// });
5861    /// # }
5862    /// ```
5863    pub fn start_child_workflow<T: Serialize>(
5864        &self,
5865        workflow_type: impl Into<String>,
5866        options: ChildWorkflowOptions,
5867        args: T,
5868    ) -> ChildWorkflowCall {
5869        ChildWorkflowCall {
5870            ctx: self.clone(),
5871            workflow_type: workflow_type.into(),
5872            options,
5873            args: Some(AvroValue::from_serialize(&args)),
5874            scheduled: false,
5875            matched_pending: false,
5876        }
5877    }
5878
5879    pub async fn start_child_workflow_avro_value<T: Serialize>(
5880        &self,
5881        workflow_type: impl Into<String>,
5882        options: ChildWorkflowOptions,
5883        args: T,
5884    ) -> Result<ChildWorkflowAvroResult> {
5885        let mut call = self.start_child_workflow(workflow_type, options, args);
5886        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5887    }
5888
5889    fn take_commands(&self) -> Result<Vec<Value>> {
5890        let mut state = self
5891            .state
5892            .lock()
5893            .map_err(|_| Error::WorkflowStatePoisoned)?;
5894        Ok(std::mem::take(&mut state.commands))
5895    }
5896
5897    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
5898        let mut state = self
5899            .state
5900            .lock()
5901            .map_err(|_| Error::WorkflowStatePoisoned)?;
5902
5903        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5904            return Err(command_mismatch(&recorded, "continue as new"));
5905        }
5906        if state.recorded_continue_as_new_sequence.is_some() {
5907            state.continue_as_new_consumed = true;
5908            return Ok(None);
5909        }
5910
5911        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
5912        let mut command = serde_json::Map::from_iter([
5913            ("type".to_string(), json!("continue_as_new")),
5914            ("arguments".to_string(), arguments),
5915            ("queue".to_string(), json!(state.task_queue.clone())),
5916        ]);
5917        if let Some(workflow_type) = request.options.workflow_type {
5918            command.insert("workflow_type".to_string(), json!(workflow_type));
5919        }
5920        if let Some(task_queue) = request.options.task_queue {
5921            command.insert("queue".to_string(), json!(task_queue));
5922        }
5923        Ok(Some(Value::Object(command)))
5924    }
5925
5926    fn matched_recorded_pending(&self) -> Result<bool> {
5927        let state = self
5928            .state
5929            .lock()
5930            .map_err(|_| Error::WorkflowStatePoisoned)?;
5931        Ok(state.matched_recorded_pending)
5932    }
5933
5934    fn ensure_history_consumed(&self) -> Result<()> {
5935        let state = self
5936            .state
5937            .lock()
5938            .map_err(|_| Error::WorkflowStatePoisoned)?;
5939        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
5940            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5941                "recorded_commands_unconsumed",
5942                Some(command.sequence()),
5943                Some(command.shape().to_string()),
5944                Some("workflow completion".to_string()),
5945                "workflow completed before consuming all recorded durable commands",
5946            )));
5947        }
5948        if let Some(sequence) = state
5949            .recorded_continue_as_new_sequence
5950            .filter(|_| !state.continue_as_new_consumed)
5951        {
5952            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5953                "recorded_continue_as_new_unconsumed",
5954                Some(sequence),
5955                Some("continue as new".to_string()),
5956                Some("workflow completion".to_string()),
5957                "workflow completed without consuming its recorded continue-as-new transition",
5958            )));
5959        }
5960        Ok(())
5961    }
5962}
5963
5964#[derive(Debug)]
5965struct WorkflowState {
5966    workflow_id: Option<String>,
5967    run_id: Option<String>,
5968    task_queue: String,
5969    payload_codec: String,
5970    history_budget: WorkflowHistoryBudget,
5971    resume_signal: Option<ResumeSignal>,
5972    recorded_commands: Vec<RecordedCommand>,
5973    recorded_continue_as_new_sequence: Option<u64>,
5974    continue_as_new_consumed: bool,
5975    command_cursor: usize,
5976    matched_recorded_pending: bool,
5977    version_markers: HashMap<String, (i32, u64)>,
5978    commands: Vec<Value>,
5979}
5980
5981impl WorkflowState {
5982    #[cfg(test)]
5983    fn new(
5984        history: Vec<HistoryEvent>,
5985        task_queue: String,
5986        payload_codec: String,
5987        resume_signal: Option<ResumeSignal>,
5988    ) -> Result<Self> {
5989        Self::new_with_identity(
5990            history,
5991            None,
5992            None,
5993            task_queue,
5994            payload_codec,
5995            resume_signal,
5996        )
5997    }
5998
5999    fn new_with_identity(
6000        history: Vec<HistoryEvent>,
6001        workflow_id: Option<String>,
6002        run_id: Option<String>,
6003        task_queue: String,
6004        payload_codec: String,
6005        resume_signal: Option<ResumeSignal>,
6006    ) -> Result<Self> {
6007        let recorded_commands = recorded_commands(
6008            &history,
6009            &payload_codec,
6010            WorkflowIdentity {
6011                workflow_id: workflow_id.clone(),
6012                run_id: run_id.clone(),
6013            },
6014        )?;
6015        let recorded_continue_as_new = history
6016            .iter()
6017            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6018            .collect::<Vec<_>>();
6019        if recorded_continue_as_new.len() > 1 {
6020            return Err(invalid_recorded_history(
6021                "duplicate_continue_as_new_transition",
6022                recorded_continue_as_new
6023                    .last()
6024                    .and_then(|event| durable_event_sequence(event))
6025                    .unwrap_or(0),
6026                "one WorkflowContinuedAsNew event",
6027                &format!(
6028                    "{} WorkflowContinuedAsNew events",
6029                    recorded_continue_as_new.len()
6030                ),
6031                "workflow history records one continue-as-new transition more than once",
6032            ));
6033        }
6034        let recorded_continue_as_new_sequence = recorded_continue_as_new
6035            .first()
6036            .map(|event| {
6037                durable_event_sequence(event).ok_or_else(|| {
6038                    Error::NonDeterministicReplay(ReplayFailure::new(
6039                        "continue_as_new_sequence_missing",
6040                        None,
6041                        Some("recorded transition sequence".to_string()),
6042                        Some("missing sequence".to_string()),
6043                        "WorkflowContinuedAsNew history is missing its recorded sequence",
6044                    ))
6045                })
6046            })
6047            .transpose()?;
6048        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6049        Ok(Self {
6050            workflow_id,
6051            run_id,
6052            task_queue,
6053            payload_codec,
6054            history_budget: WorkflowHistoryBudget {
6055                event_count,
6056                ..WorkflowHistoryBudget::default()
6057            },
6058            resume_signal,
6059            recorded_commands,
6060            recorded_continue_as_new_sequence,
6061            continue_as_new_consumed: false,
6062            command_cursor: 0,
6063            matched_recorded_pending: false,
6064            version_markers: HashMap::new(),
6065            commands: Vec::new(),
6066        })
6067    }
6068}
6069
6070#[derive(Clone, Debug)]
6071enum RecordedCommand {
6072    Activity {
6073        sequence: u64,
6074        activity_type: Option<String>,
6075        options: Option<RecordedActivityOptions>,
6076        outcome: Option<ActivityOutcome>,
6077    },
6078    Timer {
6079        sequence: u64,
6080        delay_seconds: u64,
6081        fired: bool,
6082    },
6083    ChildWorkflow {
6084        sequence: u64,
6085        workflow_type: Option<String>,
6086        outcome: Option<ChildWorkflowOutcome>,
6087    },
6088    SignalWait {
6089        sequence: u64,
6090        signal_name: String,
6091        value: Option<Vec<AvroValue>>,
6092    },
6093    SideEffect {
6094        sequence: u64,
6095        value: AvroValue,
6096    },
6097    VersionMarker {
6098        sequence: u64,
6099        change_id: String,
6100        version: i32,
6101    },
6102}
6103
6104#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6105struct RecordedActivityOptions {
6106    task_queue: RecordedSnapshotValue<Option<String>>,
6107    execution_mode: RecordedSnapshotValue<Option<String>>,
6108    retry_policy: ActivityRetrySnapshot,
6109}
6110
6111#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6112enum RecordedSnapshotValue<T> {
6113    /// Older history did not persist this field, so it cannot constrain replay.
6114    Unknown,
6115    Known(T),
6116}
6117
6118impl<T: PartialEq> RecordedSnapshotValue<T> {
6119    fn matches_current(&self, current: &Self) -> bool {
6120        match self {
6121            Self::Unknown => true,
6122            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6123        }
6124    }
6125}
6126
6127#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6128struct ActivityRetrySnapshot {
6129    snapshot_version: RecordedSnapshotValue<Option<u64>>,
6130    max_attempts: RecordedSnapshotValue<Option<u64>>,
6131    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6132    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6133    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6134    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6135    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6136    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6137}
6138
6139impl ActivityRetrySnapshot {
6140    fn matches_current(&self, current: &Self) -> bool {
6141        self.snapshot_version
6142            .matches_current(&current.snapshot_version)
6143            && self.max_attempts.matches_current(&current.max_attempts)
6144            && self
6145                .backoff_seconds
6146                .matches_current(&current.backoff_seconds)
6147            && self
6148                .start_to_close_timeout
6149                .matches_current(&current.start_to_close_timeout)
6150            && self
6151                .schedule_to_start_timeout
6152                .matches_current(&current.schedule_to_start_timeout)
6153            && self
6154                .schedule_to_close_timeout
6155                .matches_current(&current.schedule_to_close_timeout)
6156            && self
6157                .heartbeat_timeout
6158                .matches_current(&current.heartbeat_timeout)
6159            && self
6160                .non_retryable_error_types
6161                .matches_current(&current.non_retryable_error_types)
6162    }
6163}
6164
6165fn recorded_optional_u64(
6166    object: Option<&serde_json::Map<String, Value>>,
6167    field: &str,
6168) -> RecordedSnapshotValue<Option<u64>> {
6169    match object.and_then(|object| object.get(field)) {
6170        None => RecordedSnapshotValue::Unknown,
6171        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6172        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6173    }
6174}
6175
6176fn recorded_optional_string(
6177    object: &serde_json::Map<String, Value>,
6178    field: &str,
6179) -> RecordedSnapshotValue<Option<String>> {
6180    match object.get(field) {
6181        None => RecordedSnapshotValue::Unknown,
6182        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6183        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6184    }
6185}
6186
6187fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6188    let policy = policy.and_then(Value::as_object);
6189    let backoff_seconds = policy
6190        .and_then(|policy| policy.get("backoff_seconds"))
6191        .and_then(Value::as_array)
6192        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6193        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6194    let mut non_retryable_error_types = Vec::new();
6195    for error_type in policy
6196        .and_then(|policy| policy.get("non_retryable_error_types"))
6197        .and_then(Value::as_array)
6198        .into_iter()
6199        .flatten()
6200        .filter_map(Value::as_str)
6201        .map(str::trim)
6202        .filter(|error_type| !error_type.is_empty())
6203    {
6204        if !non_retryable_error_types
6205            .iter()
6206            .any(|recorded| recorded == error_type)
6207        {
6208            non_retryable_error_types.push(error_type.to_string());
6209        }
6210    }
6211
6212    ActivityRetrySnapshot {
6213        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6214        max_attempts: recorded_optional_u64(policy, "max_attempts"),
6215        backoff_seconds,
6216        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6217        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6218        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6219        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6220        non_retryable_error_types: if policy
6221            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6222        {
6223            RecordedSnapshotValue::Known(non_retryable_error_types)
6224        } else {
6225            RecordedSnapshotValue::Unknown
6226        },
6227    }
6228}
6229
6230fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6231    let policy = options.retry_policy.as_ref();
6232    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6233        Some(Value::Null) => None,
6234        Some(value) => value_as_u64(value),
6235        None => Some(1),
6236    };
6237    let backoff_seconds = policy
6238        .and_then(|policy| policy.get("backoff_seconds"))
6239        .and_then(Value::as_array)
6240        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6241        .unwrap_or_default();
6242    let non_retryable_error_types = policy
6243        .and_then(|policy| policy.get("non_retryable_error_types"))
6244        .and_then(Value::as_array)
6245        .into_iter()
6246        .flatten()
6247        .filter_map(Value::as_str)
6248        .map(str::to_string)
6249        .collect();
6250
6251    ActivityRetrySnapshot {
6252        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6253        max_attempts: RecordedSnapshotValue::Known(max_attempts),
6254        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6255        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6256        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6257        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6258        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6259        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6260    }
6261}
6262
6263fn activity_options_description(options: &RecordedActivityOptions) -> String {
6264    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6265}
6266
6267impl RecordedCommand {
6268    fn sequence(&self) -> u64 {
6269        match self {
6270            Self::Activity { sequence, .. }
6271            | Self::Timer { sequence, .. }
6272            | Self::ChildWorkflow { sequence, .. }
6273            | Self::SignalWait { sequence, .. }
6274            | Self::SideEffect { sequence, .. }
6275            | Self::VersionMarker { sequence, .. } => *sequence,
6276        }
6277    }
6278
6279    fn shape(&self) -> &'static str {
6280        match self {
6281            Self::Activity { .. } => "activity",
6282            Self::Timer { .. } => "timer",
6283            Self::ChildWorkflow { .. } => "child workflow",
6284            Self::SignalWait { .. } => "signal wait",
6285            Self::SideEffect { .. } => "side effect",
6286            Self::VersionMarker { .. } => "version marker",
6287        }
6288    }
6289}
6290
6291fn ensure_version_supported(
6292    change_id: &str,
6293    version: i32,
6294    min_supported: i32,
6295    max_supported: i32,
6296    sequence: u64,
6297) -> Result<()> {
6298    if (min_supported..=max_supported).contains(&version) {
6299        return Ok(());
6300    }
6301    Err(Error::NonDeterministicReplay(ReplayFailure::new(
6302        "version_marker_incompatible_range",
6303        (sequence != 0).then_some(sequence),
6304        Some(format!("{min_supported}..={max_supported}")),
6305        Some(format!("{change_id}:{version}")),
6306        "recorded workflow version is outside the range supported by current code",
6307    )))
6308}
6309
6310#[derive(Clone, Debug)]
6311struct ResumeSignal {
6312    signal_name: String,
6313    arguments: Vec<AvroValue>,
6314}
6315
6316pub struct ActivityCall {
6317    ctx: WorkflowContext,
6318    activity_type: String,
6319    options: ActivityOptions,
6320    args: Option<Result<AvroValue>>,
6321    scheduled: bool,
6322}
6323
6324impl ActivityCall {
6325    fn poll_avro_value(
6326        mut self: Pin<&mut Self>,
6327        _cx: &mut TaskContext<'_>,
6328    ) -> Poll<Result<AvroValue>> {
6329        let ctx = self.ctx.clone();
6330        let mut state = match ctx.state.lock() {
6331            Ok(state) => state,
6332            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6333        };
6334
6335        if self.scheduled {
6336            return Poll::Pending;
6337        }
6338
6339        let options = match self.options.validate() {
6340            Ok(options) => options,
6341            Err(error) => {
6342                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6343            }
6344        };
6345        let task_queue = options
6346            .task_queue
6347            .clone()
6348            .unwrap_or_else(|| state.task_queue.clone());
6349        let current_recorded_options = RecordedActivityOptions {
6350            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6351            // Rust schedules ordinary durable activities. The server records a
6352            // non-null mode only for a specialized execution primitive.
6353            execution_mode: RecordedSnapshotValue::Known(None),
6354            retry_policy: current_activity_retry_snapshot(&options),
6355        };
6356
6357        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6358            let sequence = recorded.sequence();
6359            match recorded {
6360                RecordedCommand::Activity {
6361                    activity_type,
6362                    options: recorded_options,
6363                    outcome,
6364                    ..
6365                } => {
6366                    if let Some(recorded_type) = activity_type {
6367                        if recorded_type != self.activity_type {
6368                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6369                                ReplayFailure::new(
6370                                    "recorded_command_detail_mismatch",
6371                                    Some(sequence),
6372                                    Some(format!("activity:{recorded_type}")),
6373                                    Some(format!("activity:{}", self.activity_type)),
6374                                    "recorded activity type differs from the current workflow command",
6375                                ),
6376                            )));
6377                        }
6378                    }
6379                    if let Some(recorded_options) = recorded_options {
6380                        if !recorded_options
6381                            .task_queue
6382                            .matches_current(&current_recorded_options.task_queue)
6383                        {
6384                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6385                                ReplayFailure::new(
6386                                    "activity_task_queue_mismatch",
6387                                    Some(sequence),
6388                                    Some(activity_options_description(&recorded_options)),
6389                                    Some(activity_options_description(&current_recorded_options)),
6390                                    "recorded activity task queue differs from the current workflow command",
6391                                ),
6392                            )));
6393                        }
6394                        if !recorded_options
6395                            .execution_mode
6396                            .matches_current(&current_recorded_options.execution_mode)
6397                        {
6398                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6399                                ReplayFailure::new(
6400                                    "activity_execution_mode_mismatch",
6401                                    Some(sequence),
6402                                    Some(activity_options_description(&recorded_options)),
6403                                    Some(activity_options_description(&current_recorded_options)),
6404                                    "recorded activity execution mode differs from the current workflow command",
6405                                ),
6406                            )));
6407                        }
6408                        if !recorded_options
6409                            .retry_policy
6410                            .matches_current(&current_recorded_options.retry_policy)
6411                        {
6412                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6413                                ReplayFailure::new(
6414                                    "activity_retry_policy_mismatch",
6415                                    Some(sequence),
6416                                    Some(activity_options_description(&recorded_options)),
6417                                    Some(activity_options_description(&current_recorded_options)),
6418                                    "recorded activity retry policy differs from the current workflow command",
6419                                ),
6420                            )));
6421                        }
6422                    }
6423                    state.command_cursor += 1;
6424                    if let Some(outcome) = outcome {
6425                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6426                    }
6427                    state.matched_recorded_pending = true;
6428                    self.scheduled = true;
6429                    return Poll::Pending;
6430                }
6431                other => {
6432                    return Poll::Ready(Err(command_mismatch(
6433                        &other,
6434                        format!("activity:{}", self.activity_type),
6435                    )));
6436                }
6437            }
6438        }
6439
6440        if !self.scheduled {
6441            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6442                Ok(args) => args,
6443                Err(error) => return Poll::Ready(Err(error)),
6444            };
6445            let arguments = normalize_avro_arguments(args);
6446            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6447                Ok(envelope) => envelope,
6448                Err(error) => return Poll::Ready(Err(error)),
6449            };
6450
6451            let mut command = serde_json::Map::from_iter([
6452                ("type".to_string(), json!("schedule_activity")),
6453                (
6454                    "activity_type".to_string(),
6455                    json!(self.activity_type.clone()),
6456                ),
6457                ("queue".to_string(), json!(task_queue)),
6458                ("arguments".to_string(), envelope),
6459            ]);
6460            for (field, value) in [
6461                ("start_to_close_timeout", options.start_to_close_timeout),
6462                (
6463                    "schedule_to_start_timeout",
6464                    options.schedule_to_start_timeout,
6465                ),
6466                (
6467                    "schedule_to_close_timeout",
6468                    options.schedule_to_close_timeout,
6469                ),
6470                ("heartbeat_timeout", options.heartbeat_timeout),
6471            ] {
6472                if let Some(value) = value {
6473                    command.insert(field.to_string(), json!(value));
6474                }
6475            }
6476            if let Some(retry_policy) = options.retry_policy {
6477                command.insert("retry_policy".to_string(), retry_policy);
6478            }
6479            state.commands.push(Value::Object(command));
6480            self.scheduled = true;
6481        }
6482
6483        Poll::Pending
6484    }
6485}
6486
6487impl Future for ActivityCall {
6488    type Output = Result<Value>;
6489
6490    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6491        match self.poll_avro_value(cx) {
6492            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6493            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6494            Poll::Pending => Poll::Pending,
6495        }
6496    }
6497}
6498
6499/// Future returned by [`WorkflowContext::sleep`].
6500pub struct TimerCall {
6501    ctx: WorkflowContext,
6502    delay_seconds: Option<u64>,
6503    scheduled: bool,
6504    matched_pending: bool,
6505}
6506
6507impl Future for TimerCall {
6508    type Output = Result<()>;
6509
6510    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6511        if self.matched_pending {
6512            return Poll::Pending;
6513        }
6514
6515        let ctx = self.ctx.clone();
6516        let Some(requested_delay) = self.delay_seconds else {
6517            return Poll::Ready(Err(Error::TimerDurationOverflow));
6518        };
6519        let mut state = match ctx.state.lock() {
6520            Ok(state) => state,
6521            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6522        };
6523
6524        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6525            match recorded {
6526                RecordedCommand::Timer {
6527                    sequence,
6528                    delay_seconds,
6529                    fired,
6530                    ..
6531                } => {
6532                    if delay_seconds != requested_delay {
6533                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6534                            ReplayFailure::new(
6535                                "timer_delay_mismatch",
6536                                Some(sequence),
6537                                Some(format!("timer:{delay_seconds}s")),
6538                                Some(format!("timer:{requested_delay}s")),
6539                                "recorded timer delay differs from the current workflow command",
6540                            ),
6541                        )));
6542                    }
6543                    state.command_cursor += 1;
6544                    if fired {
6545                        return Poll::Ready(Ok(()));
6546                    }
6547                    state.matched_recorded_pending = true;
6548                    self.scheduled = true;
6549                    self.matched_pending = true;
6550                    return Poll::Pending;
6551                }
6552                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6553            }
6554        }
6555
6556        if !self.scheduled {
6557            state.commands.push(json!({
6558                "type": "start_timer",
6559                "delay_seconds": requested_delay,
6560            }));
6561            self.scheduled = true;
6562        }
6563
6564        Poll::Pending
6565    }
6566}
6567
6568/// Future returned by [`WorkflowContext::start_child_workflow`].
6569pub struct ChildWorkflowCall {
6570    ctx: WorkflowContext,
6571    workflow_type: String,
6572    options: ChildWorkflowOptions,
6573    args: Option<Result<AvroValue>>,
6574    scheduled: bool,
6575    matched_pending: bool,
6576}
6577
6578impl ChildWorkflowCall {
6579    fn poll_avro_value(
6580        mut self: Pin<&mut Self>,
6581        _cx: &mut TaskContext<'_>,
6582    ) -> Poll<Result<ChildWorkflowAvroResult>> {
6583        if self.matched_pending {
6584            return Poll::Pending;
6585        }
6586
6587        let ctx = self.ctx.clone();
6588        let mut state = match ctx.state.lock() {
6589            Ok(state) => state,
6590            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6591        };
6592
6593        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6594            let sequence = recorded.sequence();
6595            match recorded {
6596                RecordedCommand::ChildWorkflow {
6597                    workflow_type,
6598                    outcome,
6599                    ..
6600                } => {
6601                    if let Some(recorded_type) = workflow_type {
6602                        if recorded_type != self.workflow_type {
6603                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6604                                ReplayFailure::new(
6605                                    "recorded_command_detail_mismatch",
6606                                    Some(sequence),
6607                                    Some(format!("child workflow:{recorded_type}")),
6608                                    Some(format!("child workflow:{}", self.workflow_type)),
6609                                    "recorded child workflow type differs from the current workflow command",
6610                                ),
6611                            )));
6612                        }
6613                    }
6614                    state.command_cursor += 1;
6615                    if let Some(outcome) = outcome {
6616                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6617                    }
6618                    state.matched_recorded_pending = true;
6619                    self.scheduled = true;
6620                    self.matched_pending = true;
6621                    return Poll::Pending;
6622                }
6623                other => {
6624                    return Poll::Ready(Err(command_mismatch(
6625                        &other,
6626                        format!("child workflow:{}", self.workflow_type),
6627                    )));
6628                }
6629            }
6630        }
6631
6632        if !self.scheduled {
6633            if self.options.task_queue.trim().is_empty() {
6634                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6635                    "task_queue must not be empty".to_string(),
6636                )));
6637            }
6638            for (name, value) in [
6639                (
6640                    "execution_timeout_seconds",
6641                    self.options.execution_timeout_seconds,
6642                ),
6643                ("run_timeout_seconds", self.options.run_timeout_seconds),
6644            ] {
6645                if value == Some(0) {
6646                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6647                        "{name} must be at least 1"
6648                    ))));
6649                }
6650            }
6651
6652            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6653                Ok(args) => args,
6654                Err(error) => return Poll::Ready(Err(error)),
6655            };
6656            let arguments = match encode_typed_envelope(
6657                &normalize_avro_arguments(args),
6658                &state.payload_codec,
6659            ) {
6660                Ok(arguments) => arguments,
6661                Err(error) => return Poll::Ready(Err(error)),
6662            };
6663            let mut command = json!({
6664                "type": "start_child_workflow",
6665                "workflow_type": self.workflow_type,
6666                "queue": self.options.task_queue,
6667                "parent_close_policy": self.options.parent_close_policy.as_str(),
6668                "arguments": arguments,
6669            });
6670            let object = command
6671                .as_object_mut()
6672                .expect("child workflow command is always an object");
6673            if let Some(policy) = &self.options.retry_policy {
6674                let mut retry_policy = serde_json::Map::new();
6675                if let Some(max_attempts) = policy.max_attempts {
6676                    if max_attempts == 0 {
6677                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6678                            "retry_policy.max_attempts must be at least 1".to_string(),
6679                        )));
6680                    }
6681                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6682                }
6683                if !policy.backoff_seconds.is_empty() {
6684                    retry_policy
6685                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6686                }
6687                if !policy.non_retryable_error_types.is_empty() {
6688                    retry_policy.insert(
6689                        "non_retryable_error_types".to_string(),
6690                        json!(policy.non_retryable_error_types),
6691                    );
6692                }
6693                if retry_policy.is_empty() {
6694                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6695                        "retry_policy must configure at least one field".to_string(),
6696                    )));
6697                }
6698                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6699            }
6700            if let Some(seconds) = self.options.execution_timeout_seconds {
6701                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6702            }
6703            if let Some(seconds) = self.options.run_timeout_seconds {
6704                object.insert("run_timeout_seconds".to_string(), json!(seconds));
6705            }
6706            state.commands.push(command);
6707            self.scheduled = true;
6708        }
6709
6710        Poll::Pending
6711    }
6712}
6713
6714impl Future for ChildWorkflowCall {
6715    type Output = Result<ChildWorkflowResult>;
6716
6717    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6718        match self.poll_avro_value(cx) {
6719            Poll::Ready(Ok(result)) => match result.result.into_json() {
6720                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6721                    parent: result.parent,
6722                    child: result.child,
6723                    child_workflow_type: result.child_workflow_type,
6724                    result: projected,
6725                })),
6726                Err(error) => Poll::Ready(Err(error)),
6727            },
6728            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6729            Poll::Pending => Poll::Pending,
6730        }
6731    }
6732}
6733
6734fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6735    Error::NonDeterministicReplay(ReplayFailure::new(
6736        "recorded_command_mismatch",
6737        Some(recorded.sequence()),
6738        Some(recorded.shape().to_string()),
6739        Some(actual.into()),
6740        "current workflow command does not match the recorded durable command sequence",
6741    ))
6742}
6743
6744pub struct SignalCall {
6745    ctx: WorkflowContext,
6746    signal_name: String,
6747    opened_wait: bool,
6748    matched_pending: bool,
6749}
6750
6751impl SignalCall {
6752    fn poll_avro_value(
6753        mut self: Pin<&mut Self>,
6754        _cx: &mut TaskContext<'_>,
6755    ) -> Poll<Result<Vec<AvroValue>>> {
6756        if self.matched_pending {
6757            return Poll::Pending;
6758        }
6759
6760        let ctx = self.ctx.clone();
6761        let mut state = match ctx.state.lock() {
6762            Ok(state) => state,
6763            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6764        };
6765
6766        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6767            match recorded {
6768                RecordedCommand::SignalWait {
6769                    sequence,
6770                    signal_name,
6771                    value,
6772                } => {
6773                    if signal_name != self.signal_name {
6774                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6775                            ReplayFailure::new(
6776                                "recorded_command_detail_mismatch",
6777                                Some(sequence),
6778                                Some(format!("signal wait:{signal_name}")),
6779                                Some(format!("signal wait:{}", self.signal_name)),
6780                                "recorded signal name differs from the current workflow command",
6781                            ),
6782                        )));
6783                    }
6784
6785                    state.command_cursor += 1;
6786                    if let Some(value) = value {
6787                        return Poll::Ready(Ok(value));
6788                    }
6789                    if state
6790                        .resume_signal
6791                        .as_ref()
6792                        .is_some_and(|signal| signal.signal_name == self.signal_name)
6793                    {
6794                        let signal = state
6795                            .resume_signal
6796                            .take()
6797                            .expect("matching resume signal is present");
6798                        return Poll::Ready(Ok(signal.arguments));
6799                    }
6800
6801                    state.matched_recorded_pending = true;
6802                    self.opened_wait = true;
6803                    self.matched_pending = true;
6804                    return Poll::Pending;
6805                }
6806                other => {
6807                    return Poll::Ready(Err(command_mismatch(
6808                        &other,
6809                        format!("signal wait:{}", self.signal_name),
6810                    )));
6811                }
6812            }
6813        }
6814
6815        if state
6816            .resume_signal
6817            .as_ref()
6818            .is_some_and(|signal| signal.signal_name == self.signal_name)
6819        {
6820            let signal = state
6821                .resume_signal
6822                .take()
6823                .expect("matching resume signal is present");
6824            return Poll::Ready(Ok(signal.arguments));
6825        }
6826
6827        if !self.opened_wait {
6828            state.commands.push(json!({
6829                "type": "open_signal_wait",
6830                "signal_name": self.signal_name
6831            }));
6832            self.opened_wait = true;
6833        }
6834
6835        Poll::Pending
6836    }
6837}
6838
6839impl Future for SignalCall {
6840    type Output = Result<Vec<Value>>;
6841
6842    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6843        match self.poll_avro_value(cx) {
6844            Poll::Ready(Ok(values)) => Poll::Ready(
6845                values
6846                    .into_iter()
6847                    .map(AvroValue::into_json)
6848                    .collect::<Result<Vec<_>>>(),
6849            ),
6850            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6851            Poll::Pending => Poll::Pending,
6852        }
6853    }
6854}
6855
6856#[derive(Clone, Debug)]
6857pub struct ActivityContext {
6858    client: Client,
6859    pub task_id: String,
6860    pub activity_attempt_id: String,
6861    pub lease_owner: String,
6862    pub activity_type: String,
6863    pub attempt_number: u64,
6864    pub task_queue: String,
6865    pub worker_id: String,
6866}
6867
6868impl ActivityContext {
6869    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6870        self.client
6871            .heartbeat_activity_task(
6872                &self.task_id,
6873                &self.activity_attempt_id,
6874                &self.lease_owner,
6875                details,
6876            )
6877            .await
6878    }
6879}
6880
6881fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6882    match value {
6883        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6884            value, codec,
6885        )?)),
6886        None => Ok(AvroValue::Array(Vec::new())),
6887    }
6888}
6889
6890fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
6891    let Some(signal_name) = task
6892        .signal_name
6893        .as_deref()
6894        .filter(|value| !value.is_empty())
6895    else {
6896        return Ok(None);
6897    };
6898    let Some(arguments) = task.signal_arguments.as_ref() else {
6899        return Ok(None);
6900    };
6901
6902    let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
6903    let AvroValue::Array(arguments) = decoded else {
6904        unreachable!("normalize_avro_arguments always returns an array");
6905    };
6906
6907    Ok(Some(ResumeSignal {
6908        signal_name: signal_name.to_string(),
6909        arguments,
6910    }))
6911}
6912
6913fn recorded_commands(
6914    events: &[HistoryEvent],
6915    fallback_codec: &str,
6916    parent: WorkflowIdentity,
6917) -> Result<Vec<RecordedCommand>> {
6918    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
6919    let mut last_new_sequence = None;
6920
6921    for event in events {
6922        let is_activity = matches!(
6923            event.event_type.as_str(),
6924            "ActivityScheduled"
6925                | "ActivityStarted"
6926                | "ActivityHeartbeatRecorded"
6927                | "ActivityRetryScheduled"
6928                | "ActivityCompleted"
6929                | "ActivityFailed"
6930                | "ActivityCancelled"
6931                | "ActivityTimedOut"
6932        );
6933        let is_workflow_timer = matches!(
6934            event.event_type.as_str(),
6935            "TimerScheduled" | "TimerCancelled" | "TimerFired"
6936        ) && !is_internal_timer_event(event);
6937        let is_child_workflow = matches!(
6938            event.event_type.as_str(),
6939            "ChildWorkflowScheduled"
6940                | "ChildRunCompleted"
6941                | "ChildRunFailed"
6942                | "ChildRunCancelled"
6943                | "ChildRunTerminated"
6944        );
6945        let is_signal_wait = is_recorded_signal_wait_event(event);
6946        let is_side_effect = event.event_type == "SideEffectRecorded";
6947        let is_version_marker = event.event_type == "VersionMarkerRecorded";
6948        if !is_activity
6949            && !is_workflow_timer
6950            && !is_child_workflow
6951            && !is_signal_wait
6952            && !is_side_effect
6953            && !is_version_marker
6954        {
6955            continue;
6956        }
6957
6958        let sequence = durable_event_sequence(event).ok_or_else(|| {
6959            Error::NonDeterministicReplay(ReplayFailure::new(
6960                "durable_command_sequence_missing",
6961                None,
6962                Some("positive workflow sequence".to_string()),
6963                Some(event.event_type.clone()),
6964                "durable command history event has no workflow sequence",
6965            ))
6966        })?;
6967        if sequence == 0 {
6968            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6969                "durable_command_sequence_invalid",
6970                Some(sequence),
6971                Some("positive workflow sequence".to_string()),
6972                Some(sequence.to_string()),
6973                "durable command history uses an invalid workflow sequence",
6974            )));
6975        }
6976        if !events_by_sequence.contains_key(&sequence) {
6977            if let Some(previous) = last_new_sequence {
6978                if sequence < previous {
6979                    return Err(invalid_recorded_history(
6980                        "durable_command_sequence_mismatch",
6981                        sequence,
6982                        &format!("workflow sequence greater than {previous}"),
6983                        &sequence.to_string(),
6984                        "durable commands are not strictly ordered by their recorded workflow sequence",
6985                    ));
6986                }
6987            }
6988            last_new_sequence = Some(sequence);
6989        }
6990        events_by_sequence.entry(sequence).or_default().push(event);
6991    }
6992
6993    let commands: Vec<RecordedCommand> = events_by_sequence
6994        .into_iter()
6995        .map(|(sequence, sequence_events)| {
6996            let activity_events: Vec<_> = sequence_events
6997                .iter()
6998                .copied()
6999                .filter(|event| event.event_type.starts_with("Activity"))
7000                .collect();
7001            let timer_events: Vec<_> = sequence_events
7002                .iter()
7003                .copied()
7004                .filter(|event| event.event_type.starts_with("Timer"))
7005                .collect();
7006            let child_events: Vec<_> = sequence_events
7007                .iter()
7008                .copied()
7009                .filter(|event| {
7010                    event.event_type == "ChildWorkflowScheduled"
7011                        || event.event_type.starts_with("ChildRun")
7012                })
7013                .collect();
7014            let signal_wait_events: Vec<_> = sequence_events
7015                .iter()
7016                .copied()
7017                .filter(|event| is_recorded_signal_wait_event(event))
7018                .collect();
7019            let side_effect_events: Vec<_> = sequence_events
7020                .iter()
7021                .copied()
7022                .filter(|event| event.event_type == "SideEffectRecorded")
7023                .collect();
7024            let version_marker_events: Vec<_> = sequence_events
7025                .iter()
7026                .copied()
7027                .filter(|event| event.event_type == "VersionMarkerRecorded")
7028                .collect();
7029
7030            let command_kind_count = usize::from(!activity_events.is_empty())
7031                + usize::from(!timer_events.is_empty())
7032                + usize::from(!child_events.is_empty())
7033                + usize::from(!signal_wait_events.is_empty())
7034                + usize::from(!side_effect_events.is_empty())
7035                + usize::from(!version_marker_events.is_empty());
7036            if command_kind_count > 1 {
7037                let actual = [
7038                    (!activity_events.is_empty()).then_some("activity"),
7039                    (!timer_events.is_empty()).then_some("timer"),
7040                    (!child_events.is_empty()).then_some("child workflow"),
7041                    (!signal_wait_events.is_empty()).then_some("signal wait"),
7042                    (!side_effect_events.is_empty()).then_some("side effect"),
7043                    (!version_marker_events.is_empty()).then_some("version marker"),
7044                ]
7045                .into_iter()
7046                .flatten()
7047                .collect::<Vec<_>>()
7048                .join(" and ");
7049                return Err(invalid_recorded_history(
7050                    "durable_command_sequence_collision",
7051                    sequence,
7052                    "one durable command kind",
7053                    &actual,
7054                    "one workflow sequence records more than one durable command kind",
7055                ));
7056            }
7057
7058            if !activity_events.is_empty() {
7059                let scheduled_count = activity_events
7060                    .iter()
7061                    .filter(|event| event.event_type == "ActivityScheduled")
7062                    .count();
7063                if scheduled_count > 1 {
7064                    return Err(invalid_recorded_history(
7065                        "duplicate_activity_schedule",
7066                        sequence,
7067                        "at most one ActivityScheduled event",
7068                        "multiple ActivityScheduled events",
7069                        "activity history schedules more than one command at one workflow sequence",
7070                    ));
7071                }
7072                let activity_type = activity_events.iter().find_map(|event| {
7073                    event
7074                        .payload
7075                        .get("activity_type")
7076                        .or_else(|| event.payload.get("activity_name"))
7077                        .and_then(Value::as_str)
7078                        .map(str::to_string)
7079                });
7080                if activity_events.iter().filter_map(|event| {
7081                    event
7082                        .payload
7083                        .get("activity_type")
7084                        .or_else(|| event.payload.get("activity_name"))
7085                        .and_then(Value::as_str)
7086                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7087                    return Err(invalid_recorded_history(
7088                        "activity_identity_mismatch",
7089                        sequence,
7090                        activity_type.as_deref().unwrap_or("one activity identity"),
7091                        "conflicting activity identities",
7092                        "activity lifecycle events at one workflow sequence disagree on identity",
7093                    ));
7094                }
7095                let terminal: Vec<_> = activity_events
7096                    .iter()
7097                    .copied()
7098                    .filter(|event| {
7099                        matches!(
7100                            event.event_type.as_str(),
7101                            "ActivityCompleted"
7102                                | "ActivityFailed"
7103                                | "ActivityCancelled"
7104                                | "ActivityTimedOut"
7105                        )
7106                    })
7107                    .collect();
7108                if terminal.len() > 1 {
7109                    return Err(invalid_recorded_history(
7110                        "duplicate_activity_terminal_event",
7111                        sequence,
7112                        "at most one terminal activity event",
7113                        "multiple terminal activity events",
7114                        "activity history settles one command more than once",
7115                    ));
7116                }
7117                let outcome = terminal
7118                    .first()
7119                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7120                    .transpose()?;
7121                let options = activity_events
7122                    .iter()
7123                    .find(|event| event.event_type == "ActivityScheduled")
7124                    .and_then(|event| event.payload.get("activity"))
7125                    .and_then(Value::as_object)
7126                    .map(|activity| RecordedActivityOptions {
7127                        task_queue: recorded_optional_string(activity, "queue"),
7128                        execution_mode: recorded_optional_string(activity, "execution_mode"),
7129                        retry_policy: recorded_activity_retry_snapshot(
7130                            activity.get("retry_policy"),
7131                        ),
7132                    });
7133                return Ok(RecordedCommand::Activity {
7134                    sequence,
7135                    activity_type,
7136                    options,
7137                    outcome,
7138                });
7139            }
7140
7141            if !child_events.is_empty() {
7142                let scheduled: Vec<_> = child_events
7143                    .iter()
7144                    .copied()
7145                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
7146                    .collect();
7147                if scheduled.len() != 1 {
7148                    return Err(invalid_recorded_history(
7149                        "child_workflow_schedule_missing_or_duplicate",
7150                        sequence,
7151                        "one ChildWorkflowScheduled event",
7152                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7153                        "child workflow replay requires exactly one recorded schedule event",
7154                    ));
7155                }
7156                let workflow_type = child_events.iter().find_map(|event| {
7157                    event
7158                        .payload
7159                        .get("child_workflow_type")
7160                        .or_else(|| event.payload.get("workflow_type"))
7161                        .and_then(Value::as_str)
7162                        .filter(|value| !value.is_empty())
7163                        .map(str::to_string)
7164                });
7165                if child_events
7166                    .iter()
7167                    .filter_map(|event| {
7168                        event
7169                            .payload
7170                            .get("child_workflow_type")
7171                            .or_else(|| event.payload.get("workflow_type"))
7172                            .and_then(Value::as_str)
7173                    })
7174                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
7175                {
7176                    return Err(invalid_recorded_history(
7177                        "child_workflow_identity_mismatch",
7178                        sequence,
7179                        workflow_type
7180                            .as_deref()
7181                            .unwrap_or("one child workflow type"),
7182                        "conflicting child workflow types",
7183                        "child workflow lifecycle events at one sequence disagree on type",
7184                    ));
7185                }
7186                let mut outcomes = child_workflow_outcomes(
7187                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7188                    fallback_codec,
7189                    parent.clone(),
7190                )?;
7191                if outcomes.len() > 1 {
7192                    return Err(invalid_recorded_history(
7193                        "duplicate_child_workflow_terminal_event",
7194                        sequence,
7195                        "at most one terminal child event",
7196                        "multiple terminal child events",
7197                        "child workflow history settles one command more than once",
7198                    ));
7199                }
7200                return Ok(RecordedCommand::ChildWorkflow {
7201                    sequence,
7202                    workflow_type,
7203                    outcome: outcomes.pop(),
7204                });
7205            }
7206
7207            if !signal_wait_events.is_empty() {
7208                let opened: Vec<_> = signal_wait_events
7209                    .iter()
7210                    .copied()
7211                    .filter(|event| event.event_type == "SignalWaitOpened")
7212                    .collect();
7213                if opened.len() != 1 {
7214                    return Err(invalid_recorded_history(
7215                        "signal_wait_open_missing_or_duplicate",
7216                        sequence,
7217                        "one SignalWaitOpened event",
7218                        &format!("{} SignalWaitOpened events", opened.len()),
7219                        "signal replay requires exactly one canonical wait-open event",
7220                    ));
7221                }
7222
7223                let applied: Vec<_> = signal_wait_events
7224                    .iter()
7225                    .copied()
7226                    .filter(|event| event.event_type == "SignalApplied")
7227                    .collect();
7228                if applied.len() > 1 {
7229                    return Err(invalid_recorded_history(
7230                        "duplicate_signal_wait_apply",
7231                        sequence,
7232                        "at most one SignalApplied event",
7233                        "multiple SignalApplied events",
7234                        "signal history applies one durable wait more than once",
7235                    ));
7236                }
7237
7238                let signal_names = signal_wait_events
7239                    .iter()
7240                    .map(|event| required_signal_wait_name(event, sequence))
7241                    .collect::<Result<Vec<_>>>()?;
7242                let signal_name = signal_names
7243                    .first()
7244                    .expect("signal wait events are not empty")
7245                    .clone();
7246                if signal_names.iter().any(|candidate| candidate != &signal_name) {
7247                    return Err(invalid_recorded_history(
7248                        "signal_wait_identity_mismatch",
7249                        sequence,
7250                        &signal_name,
7251                        "conflicting signal names",
7252                        "signal wait lifecycle events at one workflow sequence disagree on identity",
7253                    ));
7254                }
7255                let value = applied
7256                    .first()
7257                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
7258                    .transpose()?;
7259                return Ok(RecordedCommand::SignalWait {
7260                    sequence,
7261                    signal_name,
7262                    value,
7263                });
7264            }
7265
7266            if !side_effect_events.is_empty() {
7267                if side_effect_events.len() != 1 {
7268                    return Err(invalid_recorded_history(
7269                        "duplicate_side_effect_record",
7270                        sequence,
7271                        "one SideEffectRecorded event",
7272                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
7273                        "side-effect history records one workflow command more than once",
7274                    ));
7275                }
7276                let event = side_effect_events[0];
7277                let result = event.payload.get("result").ok_or_else(|| {
7278                    invalid_recorded_history(
7279                        "side_effect_result_missing",
7280                        sequence,
7281                        "recorded result payload",
7282                        "missing result",
7283                        "side-effect history is missing its recorded value",
7284                    )
7285                })?;
7286                let has_published_envelope = result.as_str().is_some()
7287                    || result.as_object().is_some_and(|envelope| {
7288                        envelope.get("codec").and_then(Value::as_str).is_some()
7289                            && envelope.get("blob").and_then(Value::as_str).is_some()
7290                    });
7291                if !has_published_envelope {
7292                    return Err(invalid_recorded_history(
7293                        "side_effect_payload_malformed",
7294                        sequence,
7295                        "payload blob or {codec, blob} envelope",
7296                        &result.to_string(),
7297                        "side-effect history result does not use a published payload envelope",
7298                    ));
7299                }
7300                let codec = event
7301                    .payload
7302                    .get("payload_codec")
7303                    .and_then(Value::as_str)
7304                    .unwrap_or(fallback_codec);
7305                let value = decode_wire_avro_value(result, codec).map_err(|error| {
7306                    invalid_recorded_history(
7307                        "side_effect_payload_incompatible",
7308                        sequence,
7309                        &format!("valid {codec} payload envelope"),
7310                        &error.to_string(),
7311                        "side-effect history payload cannot be decoded with its recorded codec",
7312                    )
7313                })?;
7314                return Ok(RecordedCommand::SideEffect { sequence, value });
7315            }
7316
7317            if !version_marker_events.is_empty() {
7318                if version_marker_events.len() != 1 {
7319                    return Err(invalid_recorded_history(
7320                        "duplicate_version_marker_record",
7321                        sequence,
7322                        "one VersionMarkerRecorded event",
7323                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7324                        "version-marker history records one workflow command more than once",
7325                    ));
7326                }
7327                let payload = &version_marker_events[0].payload;
7328                let change_id = payload
7329                    .get("change_id")
7330                    .and_then(Value::as_str)
7331                    .filter(|value| !value.is_empty())
7332                    .map(str::to_string)
7333                    .ok_or_else(|| {
7334                        invalid_recorded_history(
7335                            "version_marker_field_missing",
7336                            sequence,
7337                            "non-empty change_id",
7338                            "missing or invalid change_id",
7339                            "version-marker history is missing its stable change ID",
7340                        )
7341                    })?;
7342                let version = required_version_i32(payload, "version", sequence)?;
7343                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7344                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7345                if min_supported > max_supported || version < min_supported || version > max_supported {
7346                    return Err(invalid_recorded_history(
7347                        "version_marker_history_range_invalid",
7348                        sequence,
7349                        "min_supported <= version <= max_supported",
7350                        &format!("{min_supported} <= {version} <= {max_supported}"),
7351                        "recorded version marker contains an internally incompatible range",
7352                    ));
7353                }
7354                return Ok(RecordedCommand::VersionMarker {
7355                    sequence,
7356                    change_id,
7357                    version,
7358                });
7359            }
7360
7361            let scheduled: Vec<_> = timer_events
7362                .iter()
7363                .copied()
7364                .filter(|event| event.event_type == "TimerScheduled")
7365                .collect();
7366            let fired: Vec<_> = timer_events
7367                .iter()
7368                .copied()
7369                .filter(|event| event.event_type == "TimerFired")
7370                .collect();
7371            if scheduled.len() != 1 {
7372                return Err(invalid_recorded_history(
7373                    "timer_schedule_missing_or_duplicate",
7374                    sequence,
7375                    "one TimerScheduled event",
7376                    &format!("{} TimerScheduled events", scheduled.len()),
7377                    "timer replay requires exactly one recorded schedule event",
7378                ));
7379            }
7380            if fired.len() > 1 {
7381                return Err(invalid_recorded_history(
7382                    "duplicate_timer_fire",
7383                    sequence,
7384                    "at most one TimerFired event",
7385                    "multiple TimerFired events",
7386                    "timer history contains more than one fire event for a workflow sequence",
7387                ));
7388            }
7389
7390            let scheduled = scheduled[0];
7391            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7392            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7393            if let Some(fired) = fired.first() {
7394                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7395                if fired_timer_id != timer_id {
7396                    return Err(invalid_recorded_history(
7397                        "timer_identity_mismatch",
7398                        sequence,
7399                        &timer_id,
7400                        &fired_timer_id,
7401                        "TimerFired does not correspond to the recorded TimerScheduled event",
7402                    ));
7403                }
7404                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7405                if fired_delay != delay_seconds {
7406                    return Err(invalid_recorded_history(
7407                        "timer_history_delay_mismatch",
7408                        sequence,
7409                        &delay_seconds.to_string(),
7410                        &fired_delay.to_string(),
7411                        "TimerScheduled and TimerFired record different delays",
7412                    ));
7413                }
7414            }
7415
7416            Ok(RecordedCommand::Timer {
7417                sequence,
7418                delay_seconds,
7419                fired: !fired.is_empty(),
7420            })
7421        })
7422        .collect::<Result<_>>()?;
7423
7424    let mut marker_sequences = HashMap::new();
7425    for command in &commands {
7426        if let RecordedCommand::VersionMarker {
7427            sequence,
7428            change_id,
7429            ..
7430        } = command
7431        {
7432            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7433                return Err(invalid_recorded_history(
7434                    "duplicate_version_marker",
7435                    *sequence,
7436                    &format!("one marker for change ID {change_id:?}"),
7437                    &format!("markers at sequences {first_sequence} and {sequence}"),
7438                    "workflow history contains duplicate markers for one stable change ID",
7439                ));
7440            }
7441        }
7442    }
7443
7444    Ok(commands)
7445}
7446
7447fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7448    payload
7449        .get(field)
7450        .and_then(Value::as_i64)
7451        .and_then(|value| i32::try_from(value).ok())
7452        .ok_or_else(|| {
7453            invalid_recorded_history(
7454                "version_marker_field_missing",
7455                sequence,
7456                &format!("integer {field}"),
7457                "missing or out-of-range integer",
7458                "version-marker history is missing a required integer field",
7459            )
7460        })
7461}
7462
7463fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7464    event
7465        .payload
7466        .get("sequence")
7467        .or_else(|| event.payload.get("workflow_sequence"))
7468        .or_else(|| event.raw.get("sequence"))
7469        .or_else(|| event.raw.get("workflow_sequence"))
7470        .and_then(value_as_u64)
7471}
7472
7473fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7474    matches!(
7475        event
7476            .payload
7477            .get("timer_kind")
7478            .or_else(|| event.raw.get("timer_kind"))
7479            .and_then(Value::as_str),
7480        Some("condition_timeout" | "signal_timeout")
7481    )
7482}
7483
7484fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7485    event
7486        .payload
7487        .get("signal_name")
7488        .or_else(|| event.raw.get("signal_name"))
7489        .and_then(Value::as_str)
7490        .filter(|value| !value.is_empty())
7491        .map(str::to_string)
7492        .ok_or_else(|| {
7493            invalid_recorded_history(
7494                "signal_wait_name_missing",
7495                sequence,
7496                "non-empty signal_name",
7497                &event.event_type,
7498                "canonical signal-wait history is missing its signal identity",
7499            )
7500        })
7501}
7502
7503fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7504    matches!(
7505        event.event_type.as_str(),
7506        "SignalWaitOpened" | "SignalApplied"
7507    )
7508}
7509
7510fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7511    event
7512        .payload
7513        .get(field)
7514        .and_then(Value::as_str)
7515        .filter(|value| !value.is_empty())
7516        .map(str::to_string)
7517        .ok_or_else(|| {
7518            invalid_recorded_history(
7519                "timer_history_field_missing",
7520                sequence,
7521                field,
7522                &event.event_type,
7523                "timer history is missing a required identity field",
7524            )
7525        })
7526}
7527
7528fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7529    event
7530        .payload
7531        .get(field)
7532        .and_then(value_as_u64)
7533        .ok_or_else(|| {
7534            invalid_recorded_history(
7535                "timer_history_field_missing",
7536                sequence,
7537                field,
7538                &event.event_type,
7539                "timer history is missing a required numeric field",
7540            )
7541        })
7542}
7543
7544fn invalid_recorded_history(
7545    reason: &str,
7546    sequence: u64,
7547    expected: &str,
7548    actual: &str,
7549    message: &str,
7550) -> Error {
7551    Error::NonDeterministicReplay(ReplayFailure::new(
7552        reason,
7553        Some(sequence),
7554        Some(expected.to_string()),
7555        Some(actual.to_string()),
7556        message,
7557    ))
7558}
7559
7560type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7561
7562fn activity_outcome(
7563    event: &HistoryEvent,
7564    fallback_codec: &str,
7565    recorded_activity_type: Option<String>,
7566) -> Result<ActivityOutcome> {
7567    if event.event_type == "ActivityCompleted" {
7568        let codec = event
7569            .payload
7570            .get("payload_codec")
7571            .and_then(Value::as_str)
7572            .unwrap_or(fallback_codec);
7573        return Ok(Ok(decode_wire_avro_value(
7574            event.payload.get("result").unwrap_or(&Value::Null),
7575            codec,
7576        )?));
7577    }
7578
7579    let payload = &event.payload;
7580    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7581        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7582        "ActivityCancelled" => (
7583            ActivityFailureKind::Cancelled,
7584            "cancelled",
7585            "activity was cancelled",
7586        ),
7587        "ActivityTimedOut" => (
7588            ActivityFailureKind::TimedOut,
7589            "timeout",
7590            "activity timed out",
7591        ),
7592        _ => unreachable!("activity_outcome is called only for terminal activity events"),
7593    };
7594    let exception = payload
7595        .get("exception")
7596        .filter(|value| !value.is_null())
7597        .cloned();
7598    let failure_category = payload_string(payload, "failure_category");
7599    let timeout_kind = payload_string(payload, "timeout_kind");
7600    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7601        ActivityFailureKind::Failed => failure_category
7602            .clone()
7603            .unwrap_or_else(|| fallback_reason.to_string()),
7604        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7605        ActivityFailureKind::TimedOut => timeout_kind
7606            .clone()
7607            .unwrap_or_else(|| fallback_reason.to_string()),
7608    });
7609    let message = payload_string(payload, "message")
7610        .or_else(|| {
7611            exception
7612                .as_ref()
7613                .and_then(|value| payload_string(value, "message"))
7614        })
7615        .unwrap_or_else(|| fallback_message.to_string());
7616
7617    Ok(Err(ActivityFailure {
7618        kind,
7619        reason,
7620        message,
7621        activity_execution_id: payload_string(payload, "activity_execution_id"),
7622        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7623        activity_type: payload_string(payload, "activity_type")
7624            .or_else(|| payload_string(payload, "activity_name"))
7625            .or(recorded_activity_type),
7626        activity_class: payload_string(payload, "activity_class"),
7627        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7628        failure_id: payload_string(payload, "failure_id"),
7629        failure_category,
7630        timeout_kind,
7631        non_retryable: payload
7632            .get("non_retryable")
7633            .and_then(Value::as_bool)
7634            .unwrap_or(false),
7635        exception_type: payload_string(payload, "exception_type").or_else(|| {
7636            exception
7637                .as_ref()
7638                .and_then(|value| payload_string(value, "type"))
7639        }),
7640        exception_class: payload_string(payload, "exception_class").or_else(|| {
7641            exception
7642                .as_ref()
7643                .and_then(|value| payload_string(value, "class"))
7644        }),
7645        code: payload
7646            .get("code")
7647            .filter(|value| !value.is_null())
7648            .cloned(),
7649        exception,
7650    }))
7651}
7652
7653type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7654
7655fn child_workflow_outcomes(
7656    events: &[HistoryEvent],
7657    fallback_codec: &str,
7658    parent: WorkflowIdentity,
7659) -> Result<Vec<ChildWorkflowOutcome>> {
7660    let mut outcomes = Vec::new();
7661
7662    for event in events {
7663        let kind = match event.event_type.as_str() {
7664            "ChildRunCompleted" => None,
7665            "ChildRunFailed" => Some((
7666                ChildWorkflowFailureKind::Failed,
7667                "child_workflow",
7668                "child workflow failed",
7669            )),
7670            "ChildRunCancelled" => Some((
7671                ChildWorkflowFailureKind::Cancelled,
7672                "cancelled",
7673                "child workflow was cancelled",
7674            )),
7675            "ChildRunTerminated" => Some((
7676                ChildWorkflowFailureKind::Terminated,
7677                "terminated",
7678                "child workflow was terminated",
7679            )),
7680            _ => continue,
7681        };
7682        let payload = &event.payload;
7683        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7684        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7685        let child_workflow_type = payload_string(payload, "child_workflow_type");
7686
7687        if let Some((kind, reason, fallback_message)) = kind {
7688            let exception = payload
7689                .get("exception")
7690                .filter(|value| !value.is_null())
7691                .cloned();
7692            let message = payload_string(payload, "message")
7693                .or_else(|| {
7694                    exception
7695                        .as_ref()
7696                        .and_then(|value| payload_string(value, "message"))
7697                })
7698                .unwrap_or_else(|| fallback_message.to_string());
7699            let exception_type = payload_string(payload, "exception_type").or_else(|| {
7700                exception
7701                    .as_ref()
7702                    .and_then(|value| payload_string(value, "type"))
7703            });
7704            let exception_class = payload_string(payload, "exception_class").or_else(|| {
7705                exception
7706                    .as_ref()
7707                    .and_then(|value| payload_string(value, "class"))
7708            });
7709            outcomes.push(Err(ChildWorkflowFailure {
7710                kind,
7711                reason: reason.to_string(),
7712                message,
7713                parent_workflow_id: parent.workflow_id.clone(),
7714                parent_workflow_run_id: parent.run_id.clone(),
7715                child_workflow_id,
7716                child_workflow_run_id,
7717                child_workflow_type,
7718                failure_id: payload_string(payload, "failure_id"),
7719                failure_category: payload_string(payload, "failure_category"),
7720                exception_type,
7721                exception_class,
7722                non_retryable: payload
7723                    .get("non_retryable")
7724                    .and_then(Value::as_bool)
7725                    .unwrap_or(false),
7726                code: payload
7727                    .get("code")
7728                    .filter(|value| !value.is_null())
7729                    .cloned(),
7730                exception,
7731            }));
7732            continue;
7733        }
7734
7735        let codec = payload
7736            .get("payload_codec")
7737            .and_then(Value::as_str)
7738            .unwrap_or(fallback_codec);
7739        let result = payload
7740            .get("result")
7741            .or_else(|| payload.get("output"))
7742            .unwrap_or(&Value::Null);
7743        outcomes.push(Ok(ChildWorkflowAvroResult {
7744            parent: parent.clone(),
7745            child: WorkflowIdentity {
7746                workflow_id: child_workflow_id,
7747                run_id: child_workflow_run_id,
7748            },
7749            child_workflow_type,
7750            result: decode_wire_avro_value(result, codec)?,
7751        }));
7752    }
7753
7754    Ok(outcomes)
7755}
7756
7757fn payload_string(payload: &Value, key: &str) -> Option<String> {
7758    payload
7759        .get(key)
7760        .and_then(Value::as_str)
7761        .filter(|value| !value.is_empty())
7762        .map(str::to_string)
7763}
7764
7765fn workflow_failure_command(error: &Error) -> Value {
7766    let (exception_type, exception_class, properties) = match error {
7767        Error::ActivityFailed(failure) => (
7768            match failure.kind {
7769                ActivityFailureKind::Failed => "ActivityFailed",
7770                ActivityFailureKind::Cancelled => "ActivityCancelled",
7771                ActivityFailureKind::TimedOut => "ActivityTimedOut",
7772            },
7773            "durable_workflow::ActivityFailure",
7774            json!({
7775                "reason": failure.reason,
7776                "activity_execution_id": failure.activity_execution_id,
7777                "activity_attempt_id": failure.activity_attempt_id,
7778                "activity_type": failure.activity_type,
7779                "activity_class": failure.activity_class,
7780                "attempt_number": failure.attempt_number,
7781                "failure_id": failure.failure_id,
7782                "failure_category": failure.failure_category,
7783                "timeout_kind": failure.timeout_kind,
7784                "activity_non_retryable": failure.non_retryable,
7785                "activity_exception_type": failure.exception_type,
7786                "activity_exception_class": failure.exception_class,
7787                "activity_code": failure.code,
7788                "activity_exception": failure.exception,
7789            }),
7790        ),
7791        Error::ChildWorkflowFailed(failure) => (
7792            match failure.kind {
7793                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7794                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7795                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7796            },
7797            "durable_workflow::ChildWorkflowFailure",
7798            json!({
7799                "reason": failure.reason,
7800                "parent_workflow_id": failure.parent_workflow_id,
7801                "parent_workflow_run_id": failure.parent_workflow_run_id,
7802                "child_workflow_id": failure.child_workflow_id,
7803                "child_workflow_run_id": failure.child_workflow_run_id,
7804                "child_workflow_type": failure.child_workflow_type,
7805                "failure_id": failure.failure_id,
7806                "failure_category": failure.failure_category,
7807                "child_exception_type": failure.exception_type,
7808                "child_exception_class": failure.exception_class,
7809                "child_non_retryable": failure.non_retryable,
7810                "child_code": failure.code,
7811                "child_exception": failure.exception,
7812            }),
7813        ),
7814        Error::NonDeterministicReplay(_) => (
7815            "NonDeterministicReplay",
7816            "durable_workflow::Error",
7817            Value::Null,
7818        ),
7819        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7820    };
7821    let non_retryable = match error {
7822        Error::ActivityFailed(failure) => failure.non_retryable,
7823        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7824        Error::NonDeterministicReplay(_) => true,
7825        _ => false,
7826    };
7827
7828    json!({
7829        "type": "fail_workflow",
7830        "message": error.to_string(),
7831        "exception_type": exception_type,
7832        "exception_class": exception_class,
7833        "non_retryable": non_retryable,
7834        "exception": {
7835            "type": exception_type,
7836            "class": exception_class,
7837            "message": error.to_string(),
7838            "properties": properties,
7839        }
7840    })
7841}
7842
7843fn workflow_task_integrity_error(error: &Error) -> bool {
7844    matches!(
7845        error,
7846        Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7847    )
7848}
7849
7850fn decode_signal_event_arguments(
7851    event: &HistoryEvent,
7852    fallback_codec: &str,
7853) -> Result<Vec<AvroValue>> {
7854    let codec = event
7855        .payload
7856        .get("payload_codec")
7857        .and_then(Value::as_str)
7858        .unwrap_or(fallback_codec);
7859    let raw = event
7860        .payload
7861        .get("value")
7862        .or_else(|| event.payload.get("input"))
7863        .or_else(|| event.payload.get("arguments"));
7864    let decoded = match raw.filter(|value| !value.is_null()) {
7865        Some(value) => decode_wire_avro_value(value, codec)?,
7866        None => AvroValue::Array(Vec::new()),
7867    };
7868    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7869        unreachable!("normalize_avro_arguments always returns an array");
7870    };
7871    Ok(arguments)
7872}
7873
7874fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7875    let Some(export_events) = task
7876        .history_export
7877        .as_ref()
7878        .and_then(|export| export.get("history_events"))
7879        .and_then(Value::as_array)
7880    else {
7881        return Ok(());
7882    };
7883
7884    if export_events.len() > task.history_events.len() {
7885        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
7886    }
7887
7888    Ok(())
7889}
7890
7891fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7892    let Some(export) = task.history_export.as_ref() else {
7893        return Ok(());
7894    };
7895    let signals = export
7896        .get("signals")
7897        .and_then(Value::as_array)
7898        .cloned()
7899        .unwrap_or_default();
7900    let activities = export
7901        .get("activities")
7902        .and_then(Value::as_array)
7903        .cloned()
7904        .unwrap_or_default();
7905    let export_codec = export
7906        .get("payloads")
7907        .and_then(|payloads| payloads.get("codec"))
7908        .and_then(Value::as_str)
7909        .unwrap_or(&task.payload_codec)
7910        .to_string();
7911    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
7912
7913    for event in &mut task.history_events {
7914        if event.event_type == "ActivityCompleted" {
7915            let sequence = event
7916                .payload
7917                .get("sequence")
7918                .or_else(|| event.payload.get("workflow_sequence"))
7919                .and_then(value_as_u64);
7920            let Some(activity) = sequence.and_then(|sequence| {
7921                activities.iter().find(|activity| {
7922                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
7923                })
7924            }) else {
7925                continue;
7926            };
7927            let Some(payload) = event.payload.as_object_mut() else {
7928                continue;
7929            };
7930            if missing_payload(payload.get("result")) {
7931                if let Some(result) = activity
7932                    .get("result")
7933                    .filter(|value| !missing_payload(Some(value)))
7934                {
7935                    payload.insert("result".to_string(), result.clone());
7936                }
7937            }
7938            for field in ["payload_codec", "activity_type"] {
7939                if payload
7940                    .get(field)
7941                    .and_then(Value::as_str)
7942                    .unwrap_or_default()
7943                    .is_empty()
7944                {
7945                    if let Some(value) = activity.get(field) {
7946                        payload.insert(field.to_string(), value.clone());
7947                    }
7948                }
7949            }
7950            continue;
7951        }
7952
7953        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
7954            continue;
7955        }
7956        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
7957        let command_id = event
7958            .payload
7959            .get("workflow_command_id")
7960            .or_else(|| event.raw.get("workflow_command_id"))
7961            .and_then(Value::as_str);
7962        let signal_name = event
7963            .payload
7964            .get("signal_name")
7965            .and_then(Value::as_str)
7966            .unwrap_or_default()
7967            .to_string();
7968        let matched = signals
7969            .iter()
7970            .find(|signal| {
7971                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
7972            })
7973            .or_else(|| {
7974                signals.iter().find(|signal| {
7975                    command_id.is_some()
7976                        && signal.get("command_id").and_then(Value::as_str) == command_id
7977                })
7978            })
7979            .or_else(|| {
7980                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
7981                let signal = signals
7982                    .iter()
7983                    .filter(|signal| {
7984                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
7985                    })
7986                    .nth(*offset);
7987                if signal.is_some() {
7988                    *offset += 1;
7989                }
7990                signal
7991            });
7992        let Some(signal) = matched else {
7993            continue;
7994        };
7995        let signal_codec = signal
7996            .get("payload_codec")
7997            .and_then(Value::as_str)
7998            .unwrap_or(&export_codec);
7999        let Some(payload) = event.payload.as_object_mut() else {
8000            continue;
8001        };
8002        if missing_payload(payload.get("arguments")) {
8003            if let Some(arguments) = signal
8004                .get("arguments")
8005                .filter(|value| !missing_payload(Some(value)))
8006            {
8007                let envelope = match arguments {
8008                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8009                    other => other.clone(),
8010                };
8011                payload.insert("arguments".to_string(), envelope);
8012            }
8013        }
8014        if payload
8015            .get("payload_codec")
8016            .and_then(Value::as_str)
8017            .unwrap_or_default()
8018            .is_empty()
8019        {
8020            payload.insert("payload_codec".to_string(), json!(signal_codec));
8021        }
8022    }
8023
8024    Ok(())
8025}
8026
8027fn missing_payload(value: Option<&Value>) -> bool {
8028    match value {
8029        None | Some(Value::Null) => true,
8030        Some(Value::String(value)) => value.is_empty(),
8031        Some(_) => false,
8032    }
8033}
8034
8035fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8036    let export_signals = task
8037        .history_export
8038        .as_ref()
8039        .and_then(|export| export.get("signals"))
8040        .and_then(Value::as_array)
8041        .cloned()
8042        .unwrap_or_default();
8043    let export_codec = task
8044        .history_export
8045        .as_ref()
8046        .and_then(|export| export.get("payloads"))
8047        .and_then(|payloads| payloads.get("codec"))
8048        .and_then(Value::as_str)
8049        .unwrap_or(&task.payload_codec);
8050    let mut name_offsets: HashMap<String, usize> = HashMap::new();
8051    let mut signals = Vec::new();
8052
8053    for event in &task.history_events {
8054        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8055            continue;
8056        }
8057
8058        let name = event
8059            .payload
8060            .get("signal_name")
8061            .and_then(Value::as_str)
8062            .unwrap_or_default();
8063        if name.is_empty() {
8064            continue;
8065        }
8066        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8067        let command_id = event
8068            .payload
8069            .get("workflow_command_id")
8070            .or_else(|| event.raw.get("workflow_command_id"))
8071            .and_then(Value::as_str);
8072        let matched_export = export_signals
8073            .iter()
8074            .find(|candidate| {
8075                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8076            })
8077            .or_else(|| {
8078                export_signals.iter().find(|candidate| {
8079                    command_id.is_some()
8080                        && candidate.get("command_id").and_then(Value::as_str) == command_id
8081                })
8082            })
8083            .or_else(|| {
8084                let offset = name_offsets.entry(name.to_string()).or_default();
8085                let candidate = export_signals
8086                    .iter()
8087                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8088                    .nth(*offset);
8089                if candidate.is_some() {
8090                    *offset += 1;
8091                }
8092                candidate
8093            });
8094        let codec = event
8095            .payload
8096            .get("payload_codec")
8097            .and_then(Value::as_str)
8098            .or_else(|| {
8099                matched_export
8100                    .and_then(|signal| signal.get("payload_codec"))
8101                    .and_then(Value::as_str)
8102            })
8103            .unwrap_or(export_codec);
8104        let raw_arguments = event
8105            .payload
8106            .get("value")
8107            .or_else(|| event.payload.get("input"))
8108            .or_else(|| event.payload.get("arguments"))
8109            .filter(|value| !value.is_null())
8110            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8111        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8112        let workflow_sequence = event
8113            .payload
8114            .get("workflow_sequence")
8115            .and_then(value_as_u64)
8116            .or_else(|| {
8117                matched_export
8118                    .and_then(|signal| signal.get("workflow_sequence"))
8119                    .and_then(value_as_u64)
8120            });
8121
8122        signals.push(QuerySignal {
8123            id: signal_id.map(str::to_string).or_else(|| {
8124                matched_export
8125                    .and_then(|signal| signal.get("id"))
8126                    .and_then(Value::as_str)
8127                    .map(str::to_string)
8128            }),
8129            name: name.to_string(),
8130            arguments,
8131            avro_arguments,
8132            workflow_sequence,
8133        });
8134    }
8135
8136    if signals.is_empty() {
8137        for signal in export_signals {
8138            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8139                continue;
8140            }
8141            let Some(name) = signal.get("name").and_then(Value::as_str) else {
8142                continue;
8143            };
8144            let codec = signal
8145                .get("payload_codec")
8146                .and_then(Value::as_str)
8147                .unwrap_or(export_codec);
8148            let (arguments, avro_arguments) =
8149                decode_query_signal_arguments(signal.get("arguments"), codec)?;
8150            signals.push(QuerySignal {
8151                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8152                name: name.to_string(),
8153                arguments,
8154                avro_arguments,
8155                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8156            });
8157        }
8158        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8159    }
8160
8161    Ok(signals)
8162}
8163
8164fn decode_query_signal_arguments(
8165    raw: Option<&Value>,
8166    codec: &str,
8167) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8168    let decoded = match raw.filter(|value| !value.is_null()) {
8169        Some(value) => decode_wire_avro_value(value, codec)?,
8170        None => AvroValue::Array(Vec::new()),
8171    };
8172    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8173        unreachable!("normalize_avro_arguments always returns an array");
8174    };
8175    let arguments = avro_arguments
8176        .iter()
8177        .cloned()
8178        .map(AvroValue::into_json)
8179        .collect::<Result<Vec<_>>>()?;
8180    Ok((arguments, avro_arguments))
8181}
8182
8183fn value_as_u64(value: &Value) -> Option<u64> {
8184    value
8185        .as_u64()
8186        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8187}
8188
8189#[cfg(test)]
8190mod tests {
8191    use super::*;
8192    use std::{
8193        io::{Read, Write},
8194        net::{SocketAddr, TcpListener, TcpStream},
8195        sync::atomic::AtomicUsize,
8196        thread,
8197    };
8198
8199    fn typed_fidelity_probe() -> AvroValue {
8200        AvroValue::Map(BTreeMap::from([
8201            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8202            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8203            (
8204                "numeric".to_string(),
8205                AvroValue::Map(BTreeMap::from([
8206                    ("0".to_string(), AvroValue::String("zero".to_string())),
8207                    ("1".to_string(), AvroValue::String("one".to_string())),
8208                ])),
8209            ),
8210            (
8211                "nested".to_string(),
8212                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8213                    "enabled".to_string(),
8214                    AvroValue::Boolean(true),
8215                )]))]),
8216            ),
8217            (
8218                "projection_collisions".to_string(),
8219                AvroValue::Array(projection_collision_probe()),
8220            ),
8221        ]))
8222    }
8223
8224    fn projection_collision_probe() -> Vec<AvroValue> {
8225        vec![
8226            AvroValue::Map(BTreeMap::from([
8227                ("$type".to_string(), AvroValue::String("bytes".to_string())),
8228                (
8229                    "base64".to_string(),
8230                    AvroValue::String("ordinary user text".to_string()),
8231                ),
8232            ])),
8233            AvroValue::Map(BTreeMap::from([
8234                ("$type".to_string(), AvroValue::String("map".to_string())),
8235                (
8236                    "entries".to_string(),
8237                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8238                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
8239                        (
8240                            "value".to_string(),
8241                            AvroValue::String("user map".to_string()),
8242                        ),
8243                    ]))]),
8244                ),
8245            ])),
8246        ]
8247    }
8248
8249    #[derive(Clone, Debug, Default, PartialEq)]
8250    struct ReplayCounterState {
8251        loaded: Option<String>,
8252        count: i64,
8253        finished: bool,
8254    }
8255
8256    fn replay_counter_worker() -> Worker {
8257        let client = Client::new("http://127.0.0.1:8080").expect("client");
8258        let mut worker = Worker::new(client, "rust-workers");
8259        worker.register_replayed_workflow(
8260            "replay-counter",
8261            ReplayCounterState::default,
8262            |ctx, _input, state| async move {
8263                let loaded = ctx.activity("load-counter", json!([])).await?;
8264                state.update(|current| {
8265                    current.loaded = loaded.as_str().map(str::to_string);
8266                })?;
8267                for _ in 0..2 {
8268                    let signal = ctx.wait_signal("increment").await?;
8269                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8270                    state.update(|current| current.count += amount)?;
8271                }
8272                state.update(|current| current.finished = true)?;
8273                state.read(|current| Ok(json!(current.count)))?
8274            },
8275        );
8276        worker.register_replayed_query::<ReplayCounterState, _, _>(
8277            "replay-counter",
8278            "current",
8279            |_ctx, state, _args| async move {
8280                Ok(json!({
8281                    "loaded": state.loaded,
8282                    "count": state.count,
8283                    "finished": state.finished,
8284                }))
8285            },
8286        );
8287        worker.register_replayed_query::<ReplayCounterState, _, _>(
8288            "replay-counter",
8289            "detached-mutation",
8290            |_ctx, state, _args| async move {
8291                let mut detached = (*state).clone();
8292                detached.count = 999;
8293                Ok(json!(detached.count))
8294            },
8295        );
8296        worker.register_replayed_query::<ReplayCounterState, _, _>(
8297            "replay-counter",
8298            "failed-mutation",
8299            |_ctx, state, _args| async move {
8300                let mut detached = (*state).clone();
8301                detached.count = 999;
8302                Err(Error::WorkerLoop("query refused".to_string()))
8303            },
8304        );
8305        worker
8306    }
8307
8308    fn replay_counter_query(
8309        query_name: &str,
8310        history_events: Value,
8311        run_status: &str,
8312    ) -> QueryTask {
8313        serde_json::from_value(json!({
8314            "query_task_id": format!("query-{query_name}"),
8315            "workflow_type": "replay-counter",
8316            "query_name": query_name,
8317            "payload_codec": "json",
8318            "workflow_arguments": {"codec": "json", "blob": "[]"},
8319            "query_arguments": {"codec": "json", "blob": "[]"},
8320            "history_events": history_events,
8321            "run_status": run_status,
8322        }))
8323        .expect("query task")
8324    }
8325
8326    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8327        workflow_context_with_codec(history, JSON_CODEC)
8328    }
8329
8330    fn workflow_context_with_codec(
8331        history: Vec<HistoryEvent>,
8332        payload_codec: &str,
8333    ) -> WorkflowContext {
8334        WorkflowContext {
8335            state: Arc::new(Mutex::new(
8336                WorkflowState::new_with_identity(
8337                    history,
8338                    None,
8339                    None,
8340                    "rust-workers".to_string(),
8341                    payload_codec.to_string(),
8342                    None,
8343                )
8344                .expect("valid workflow history"),
8345            )),
8346        }
8347    }
8348
8349    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8350        HistoryEvent {
8351            event_type: event_type.to_string(),
8352            payload,
8353            raw: HashMap::new(),
8354        }
8355    }
8356
8357    fn workflow_task(
8358        workflow_type: &str,
8359        history_events: Vec<HistoryEvent>,
8360        payload_codec: &str,
8361    ) -> WorkflowTask {
8362        WorkflowTask {
8363            task_id: format!("wft-{workflow_type}"),
8364            workflow_id: Some(format!("wf-{workflow_type}")),
8365            run_id: Some(format!("run-{workflow_type}")),
8366            workflow_type: workflow_type.to_string(),
8367            payload_codec: payload_codec.to_string(),
8368            arguments: Some(
8369                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8370            ),
8371            total_history_events: Some(history_events.len() as u64),
8372            history_size_bytes: None,
8373            continue_as_new_recommended: None,
8374            history_budget_pressure: None,
8375            history_events,
8376            next_history_page_token: None,
8377            workflow_task_attempt: 1,
8378            workflow_signal_id: None,
8379            signal_name: None,
8380            signal_arguments: None,
8381            workflow_update_id: None,
8382            update_name: None,
8383            lease_owner: Some("rust-worker".to_string()),
8384        }
8385    }
8386
8387    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8388    struct SideEffectProbe {
8389        request_id: String,
8390        attempt: u32,
8391    }
8392
8393    #[test]
8394    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8395        let calls = AtomicUsize::new(0);
8396        let ctx = workflow_context(Vec::new());
8397        let value = ctx
8398            .side_effect(|| {
8399                calls.fetch_add(1, Ordering::SeqCst);
8400                SideEffectProbe {
8401                    request_id: "request-42".to_string(),
8402                    attempt: 3,
8403                }
8404            })
8405            .expect("first side effect");
8406        assert_eq!(value.attempt, 3);
8407        assert_eq!(calls.load(Ordering::SeqCst), 1);
8408        let commands = ctx.take_commands().expect("commands");
8409        assert_eq!(commands.len(), 1);
8410        assert_eq!(commands[0]["type"], "record_side_effect");
8411        assert_eq!(
8412            decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8413            serde_json::to_value(&value).expect("value")
8414        );
8415
8416        let replay = workflow_context(vec![history_event(
8417            "SideEffectRecorded",
8418            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8419        )]);
8420        let replayed: SideEffectProbe = replay
8421            .side_effect(|| {
8422                calls.fetch_add(1, Ordering::SeqCst);
8423                panic!("committed side-effect callbacks must not run during replay")
8424            })
8425            .expect("replayed side effect");
8426        assert_eq!(replayed, value);
8427        assert_eq!(calls.load(Ordering::SeqCst), 1);
8428        assert!(replay.take_commands().expect("commands").is_empty());
8429        replay.ensure_history_consumed().expect("history consumed");
8430    }
8431
8432    #[test]
8433    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8434        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8435        let value = ctx
8436            .side_effect(|| SideEffectProbe {
8437                request_id: "avro-request".to_string(),
8438                attempt: 1,
8439            })
8440            .expect("Avro side effect");
8441        let uuid = ctx.uuid_v4().expect("deterministic UUID");
8442        let commands = ctx.take_commands().expect("commands");
8443        assert_eq!(commands.len(), 2);
8444        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8445        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8446        assert_eq!(
8447            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8448            serde_json::to_value(&value).expect("value")
8449        );
8450
8451        let replay = workflow_context_with_codec(
8452            vec![
8453                history_event(
8454                    "SideEffectRecorded",
8455                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8456                ),
8457                history_event(
8458                    "SideEffectRecorded",
8459                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8460                ),
8461            ],
8462            DEFAULT_CODEC,
8463        );
8464        let replayed: SideEffectProbe = replay
8465            .side_effect(|| panic!("Avro callback must not run"))
8466            .expect("replayed Avro value");
8467        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8468        assert_eq!(replayed, value);
8469        assert_eq!(replayed_uuid, uuid);
8470        assert!(replay.take_commands().expect("commands").is_empty());
8471    }
8472
8473    #[test]
8474    fn typed_side_effect_replay_preserves_bytes_and_maps() {
8475        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8476        let value = ctx
8477            .side_effect_avro_value(typed_fidelity_probe)
8478            .expect("typed side effect");
8479        let commands = ctx.take_commands().expect("side-effect command");
8480        assert_eq!(
8481            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8482                .expect("recorded side effect"),
8483            value
8484        );
8485
8486        let replay = workflow_context_with_codec(
8487            vec![history_event(
8488                "SideEffectRecorded",
8489                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8490            )],
8491            DEFAULT_CODEC,
8492        );
8493        assert_eq!(
8494            replay
8495                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8496                .expect("replayed typed side effect"),
8497            value
8498        );
8499    }
8500
8501    #[test]
8502    fn ordered_side_effects_share_the_durable_command_stream() {
8503        let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8504        let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8505        let ctx = workflow_context(vec![
8506            history_event(
8507                "SideEffectRecorded",
8508                json!({"sequence": 1, "result": first}),
8509            ),
8510            history_event(
8511                "SideEffectRecorded",
8512                json!({"sequence": 2, "result": second}),
8513            ),
8514        ]);
8515        let first: String = ctx
8516            .side_effect(|| panic!("first callback must not run"))
8517            .expect("first replay");
8518        let second: i32 = ctx
8519            .side_effect(|| panic!("second callback must not run"))
8520            .expect("second replay");
8521        assert_eq!(first, "first");
8522        assert_eq!(second, 29);
8523        ctx.ensure_history_consumed().expect("ordered history");
8524
8525        let reordered = workflow_context(vec![history_event(
8526            "VersionMarkerRecorded",
8527            json!({
8528                "sequence": 1,
8529                "change_id": "before-side-effect",
8530                "version": 1,
8531                "min_supported": 1,
8532                "max_supported": 1,
8533            }),
8534        )]);
8535        let error = reordered
8536            .side_effect(|| "new".to_string())
8537            .expect_err("command reordering must fail");
8538        assert!(matches!(
8539            error,
8540            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8541                if reason == "recorded_command_mismatch"
8542        ));
8543    }
8544
8545    #[test]
8546    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8547        let ctx = workflow_context(Vec::new());
8548        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8549        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8550        assert!(ctx.patched("new-search").expect("patch"));
8551        ctx.deprecate_patch("new-search").expect("deprecate patch");
8552        let commands = ctx.take_commands().expect("commands");
8553        assert_eq!(commands.len(), 2);
8554        assert_eq!(commands[0]["type"], "record_version_marker");
8555        assert_eq!(commands[0]["version"], 2);
8556        assert_eq!(commands[1]["change_id"], "new-search");
8557
8558        let replay = workflow_context(vec![history_event(
8559            "VersionMarkerRecorded",
8560            json!({
8561                "sequence": 1,
8562                "change_id": "checkout-v2",
8563                "version": 2,
8564                "min_supported": 1,
8565                "max_supported": 2,
8566            }),
8567        )]);
8568        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8569        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8570        assert!(replay.take_commands().expect("commands").is_empty());
8571        replay.ensure_history_consumed().expect("history consumed");
8572    }
8573
8574    #[test]
8575    fn version_markers_reject_incompatible_or_malformed_history() {
8576        let incompatible = workflow_context(vec![history_event(
8577            "VersionMarkerRecorded",
8578            json!({
8579                "sequence": 1,
8580                "change_id": "checkout-v2",
8581                "version": 1,
8582                "min_supported": 1,
8583                "max_supported": 2,
8584            }),
8585        )]);
8586        let error = incompatible
8587            .get_version("checkout-v2", 2, 3)
8588            .expect_err("old version is unsupported");
8589        assert!(matches!(
8590            error,
8591            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8592                if reason == "version_marker_incompatible_range"
8593        ));
8594
8595        for (history, reason) in [
8596            (
8597                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8598                "side_effect_result_missing",
8599            ),
8600            (
8601                vec![history_event(
8602                    "SideEffectRecorded",
8603                    json!({
8604                        "sequence": 1,
8605                        "result": {"codec": "avro", "blob": "not-base64"},
8606                    }),
8607                )],
8608                "side_effect_payload_incompatible",
8609            ),
8610            (
8611                vec![history_event(
8612                    "SideEffectRecorded",
8613                    json!({"sequence": 1, "result": {"unwrapped": true}}),
8614                )],
8615                "side_effect_payload_malformed",
8616            ),
8617            (
8618                vec![history_event(
8619                    "VersionMarkerRecorded",
8620                    json!({
8621                        "sequence": 1,
8622                        "change_id": "change",
8623                        "version": 1,
8624                        "min_supported": 2,
8625                        "max_supported": 1,
8626                    }),
8627                )],
8628                "version_marker_history_range_invalid",
8629            ),
8630        ] {
8631            let error = WorkflowState::new(
8632                history,
8633                "rust-workers".to_string(),
8634                JSON_CODEC.to_string(),
8635                None,
8636            )
8637            .expect_err("malformed history must fail");
8638            assert!(matches!(
8639                error,
8640                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8641                    if actual == reason
8642            ));
8643        }
8644    }
8645
8646    #[test]
8647    fn duplicate_side_effects_and_version_markers_are_rejected() {
8648        let duplicate_side_effect = WorkflowState::new(
8649            vec![
8650                history_event(
8651                    "SideEffectRecorded",
8652                    json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8653                ),
8654                history_event(
8655                    "SideEffectRecorded",
8656                    json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8657                ),
8658            ],
8659            "rust-workers".to_string(),
8660            JSON_CODEC.to_string(),
8661            None,
8662        )
8663        .expect_err("duplicate side effect");
8664        assert!(matches!(
8665            duplicate_side_effect,
8666            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8667                if reason == "duplicate_side_effect_record"
8668        ));
8669
8670        let marker = |sequence| {
8671            history_event(
8672                "VersionMarkerRecorded",
8673                json!({
8674                    "sequence": sequence,
8675                    "change_id": "same-change",
8676                    "version": 1,
8677                    "min_supported": 1,
8678                    "max_supported": 1,
8679                }),
8680            )
8681        };
8682        let duplicate_marker = WorkflowState::new(
8683            vec![marker(1), marker(3)],
8684            "rust-workers".to_string(),
8685            JSON_CODEC.to_string(),
8686            None,
8687        )
8688        .expect_err("duplicate marker");
8689        assert!(matches!(
8690            duplicate_marker,
8691            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8692                if reason == "duplicate_version_marker"
8693        ));
8694    }
8695
8696    #[test]
8697    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8698        fn worker(calls: Arc<AtomicUsize>) -> Worker {
8699            let client = Client::new("http://127.0.0.1:8080").expect("client");
8700            let mut worker = Worker::new(client, "rust-workers");
8701            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8702                let calls = Arc::clone(&calls);
8703                async move {
8704                    let captured = ctx.side_effect(|| {
8705                        calls.fetch_add(1, Ordering::SeqCst);
8706                        "captured-once".to_string()
8707                    })?;
8708                    let version = ctx.get_version("cold-restart", 1, 2)?;
8709                    Ok(json!({"captured": captured, "version": version}))
8710                }
8711            });
8712            worker
8713        }
8714
8715        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8716            WorkflowTask {
8717                task_id: "wft-side-effect-version".to_string(),
8718                workflow_id: Some("wf-side-effect-version".to_string()),
8719                run_id: Some("run-side-effect-version".to_string()),
8720                workflow_type: "rust.side-effect-version".to_string(),
8721                payload_codec: JSON_CODEC.to_string(),
8722                arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8723                history_events,
8724                total_history_events: None,
8725                history_size_bytes: None,
8726                continue_as_new_recommended: None,
8727                history_budget_pressure: None,
8728                next_history_page_token: None,
8729                workflow_task_attempt: 1,
8730                workflow_signal_id: None,
8731                signal_name: None,
8732                signal_arguments: None,
8733                workflow_update_id: None,
8734                update_name: None,
8735                lease_owner: Some("rust-worker".to_string()),
8736            }
8737        }
8738
8739        let calls = Arc::new(AtomicUsize::new(0));
8740        let initial = worker(Arc::clone(&calls))
8741            .execute_workflow_task(task(Vec::new()))
8742            .expect("initial execution");
8743        assert_eq!(
8744            initial
8745                .iter()
8746                .map(|command| &command["type"])
8747                .collect::<Vec<_>>(),
8748            vec![
8749                "record_side_effect",
8750                "record_version_marker",
8751                "complete_workflow"
8752            ]
8753        );
8754        assert_eq!(calls.load(Ordering::SeqCst), 1);
8755
8756        let restarted = worker(Arc::clone(&calls));
8757        let replayed = restarted
8758            .execute_workflow_task(task(vec![
8759                history_event(
8760                    "SideEffectRecorded",
8761                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8762                ),
8763                history_event(
8764                    "VersionMarkerRecorded",
8765                    json!({
8766                        "sequence": 2,
8767                        "change_id": "cold-restart",
8768                        "version": 2,
8769                        "min_supported": 1,
8770                        "max_supported": 2,
8771                    }),
8772                ),
8773            ]))
8774            .expect("cold replay");
8775        assert_eq!(replayed.len(), 1);
8776        assert_eq!(replayed[0]["type"], "complete_workflow");
8777        assert_eq!(calls.load(Ordering::SeqCst), 1);
8778    }
8779
8780    #[test]
8781    fn side_effect_replay_rejects_changed_rust_value_type() {
8782        let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8783        let ctx = workflow_context(vec![history_event(
8784            "SideEffectRecorded",
8785            json!({"sequence": 1, "result": result}),
8786        )]);
8787        let error = ctx
8788            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8789            .expect_err("changed type must fail replay");
8790        assert!(matches!(
8791            error,
8792            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8793                if reason == "side_effect_type_mismatch"
8794        ));
8795    }
8796
8797    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8798        vec![
8799            history_event(
8800                "ActivityScheduled",
8801                json!({
8802                    "sequence": 1,
8803                    "activity_type": "flaky",
8804                    "activity_execution_id": "act-1",
8805                    "activity": {
8806                        "id": "act-1",
8807                        "sequence": 1,
8808                        "type": "flaky",
8809                        "queue": "critical-activities",
8810                        "execution_mode": null,
8811                        "retry_policy": {
8812                            "snapshot_version": 1,
8813                            "max_attempts": 3,
8814                            "backoff_seconds": [2, 4],
8815                            "start_to_close_timeout": 30,
8816                            "schedule_to_start_timeout": 5,
8817                            "schedule_to_close_timeout": 90,
8818                            "heartbeat_timeout": 10,
8819                            "non_retryable_error_types": ["PermanentError"]
8820                        }
8821                    }
8822                }),
8823            ),
8824            history_event(
8825                "ActivityStarted",
8826                json!({
8827                    "sequence": 1,
8828                    "activity_type": "flaky",
8829                    "activity_execution_id": "act-1",
8830                    "activity_attempt_id": "attempt-1",
8831                    "attempt_number": 1
8832                }),
8833            ),
8834            history_event(
8835                "ActivityRetryScheduled",
8836                json!({
8837                    "sequence": 1,
8838                    "activity_type": "flaky",
8839                    "activity_execution_id": "act-1",
8840                    "activity_attempt_id": "attempt-1",
8841                    "attempt_number": 1,
8842                    "retry_after_attempt": 1,
8843                    "retry_backoff_seconds": 2,
8844                    "failure_category": "activity",
8845                    "exception_type": "TransientError"
8846                }),
8847            ),
8848            history_event(
8849                "ActivityStarted",
8850                json!({
8851                    "sequence": 1,
8852                    "activity_type": "flaky",
8853                    "activity_execution_id": "act-1",
8854                    "activity_attempt_id": "attempt-2",
8855                    "attempt_number": 2
8856                }),
8857            ),
8858            history_event(
8859                "ActivityCompleted",
8860                json!({
8861                    "sequence": 1,
8862                    "activity_type": "flaky",
8863                    "activity_execution_id": "act-1",
8864                    "activity_attempt_id": "attempt-2",
8865                    "attempt_number": 2,
8866                    "payload_codec": "json",
8867                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
8868                }),
8869            ),
8870        ]
8871    }
8872
8873    fn retry_activity_options() -> ActivityOptions {
8874        ActivityOptions::new()
8875            .task_queue("critical-activities")
8876            .retry_policy(
8877                ActivityRetryPolicy::new(3)
8878                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
8879                    .non_retryable_error_type("PermanentError"),
8880            )
8881            .start_to_close_timeout(Duration::from_secs(30))
8882            .schedule_to_start_timeout(Duration::from_secs(5))
8883            .schedule_to_close_timeout(Duration::from_secs(90))
8884            .heartbeat_timeout(Duration::from_secs(10))
8885    }
8886
8887    #[test]
8888    fn fixed_avro_value_round_trips_json_values() {
8889        let value = json!({"greeting": "hello", "count": 3, "ok": true});
8890        let envelope = PayloadEnvelope::avro(&value).expect("encode");
8891        assert_eq!(envelope.codec, DEFAULT_CODEC);
8892        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
8893    }
8894
8895    #[tokio::test]
8896    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
8897        let client = Client::new("http://127.0.0.1:8080").expect("client");
8898        let mut worker = Worker::new(client, "rust-workers");
8899        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
8900        worker
8901            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
8902        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
8903            Ok(input)
8904        });
8905        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
8906            Ok(input)
8907        });
8908        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
8909            Ok(AvroValue::Array(
8910                ctx.wait_signal_avro_value("changed").await?,
8911            ))
8912        });
8913
8914        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
8915        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
8916
8917        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
8918        workflow.arguments = Some(envelope.clone());
8919        let commands = worker
8920            .execute_workflow_task(workflow)
8921            .expect("typed workflow task");
8922        assert_eq!(commands[0]["type"], "complete_workflow");
8923        assert_eq!(
8924            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8925                .expect("typed workflow result"),
8926            arguments
8927        );
8928
8929        let activity = ActivityTask {
8930            task_id: "activity-typed".to_string(),
8931            activity_attempt_id: Some("attempt-typed".to_string()),
8932            attempt_id: None,
8933            activity_type: "typed.activity".to_string(),
8934            payload_codec: DEFAULT_CODEC.to_string(),
8935            arguments: Some(envelope.clone()),
8936            attempt_number: 1,
8937            lease_owner: Some("rust-worker".to_string()),
8938        };
8939        assert_eq!(
8940            worker
8941                .execute_activity_task(activity)
8942                .await
8943                .expect("typed activity result"),
8944            arguments
8945        );
8946
8947        let query = QueryTask {
8948            query_task_id: "query-typed".to_string(),
8949            query_task_attempt: 1,
8950            lease_owner: Some("rust-worker".to_string()),
8951            workflow_id: Some("typed-1".to_string()),
8952            run_id: Some("run-typed".to_string()),
8953            workflow_type: "typed.echo".to_string(),
8954            query_name: "inspect".to_string(),
8955            payload_codec: DEFAULT_CODEC.to_string(),
8956            workflow_arguments: Some(
8957                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
8958                    .expect("workflow input"),
8959            ),
8960            query_arguments: Some(envelope.clone()),
8961            history_events: Vec::new(),
8962            history_export: None,
8963            run_status: Some("running".to_string()),
8964        };
8965        assert_eq!(
8966            worker
8967                .execute_query_task(query)
8968                .await
8969                .expect("typed query result"),
8970            arguments
8971        );
8972
8973        let mut update = workflow_task(
8974            "typed.echo",
8975            vec![history_event(
8976                "UpdateAccepted",
8977                json!({
8978                    "update_id": "update-typed",
8979                    "update_name": "replace",
8980                    "arguments": envelope.clone(),
8981                }),
8982            )],
8983            DEFAULT_CODEC,
8984        );
8985        update.workflow_update_id = Some("update-typed".to_string());
8986        update.update_name = Some("replace".to_string());
8987        let commands = worker
8988            .execute_workflow_task(update)
8989            .expect("typed update task");
8990        assert_eq!(commands[0]["type"], "complete_update");
8991        assert_eq!(
8992            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8993                .expect("typed update result"),
8994            arguments
8995        );
8996
8997        let mut signal = workflow_task(
8998            "typed.signal",
8999            vec![history_event(
9000                "SignalReceived",
9001                json!({
9002                    "signal_id": "signal-typed",
9003                    "signal_name": "changed",
9004                    "arguments": envelope.clone(),
9005                }),
9006            )],
9007            DEFAULT_CODEC,
9008        );
9009        signal.workflow_signal_id = Some("signal-typed".to_string());
9010        signal.signal_name = Some("changed".to_string());
9011        signal.signal_arguments = Some(envelope);
9012        let commands = worker
9013            .execute_workflow_task(signal)
9014            .expect("typed signal resume");
9015        assert_eq!(
9016            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9017                .expect("typed signal result"),
9018            arguments
9019        );
9020    }
9021
9022    #[tokio::test]
9023    async fn typed_helpers_never_parse_json_inspection_projection() {
9024        let collision_values = projection_collision_probe();
9025        let expected = AvroValue::Array(collision_values.clone());
9026        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9027
9028        let activity_context = workflow_context_with_codec(
9029            vec![history_event(
9030                "ActivityCompleted",
9031                json!({
9032                    "sequence": 1,
9033                    "activity_type": "collision.activity",
9034                    "payload_codec": DEFAULT_CODEC,
9035                    "result": envelope.clone(),
9036                }),
9037            )],
9038            DEFAULT_CODEC,
9039        );
9040        assert_eq!(
9041            activity_context
9042                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9043                .await
9044                .expect("typed activity collision result"),
9045            expected
9046        );
9047
9048        let signal_context = workflow_context_with_codec(
9049            vec![
9050                history_event(
9051                    "SignalWaitOpened",
9052                    json!({"sequence": 1, "signal_name": "collision"}),
9053                ),
9054                history_event(
9055                    "SignalApplied",
9056                    json!({
9057                        "sequence": 1,
9058                        "signal_name": "collision",
9059                        "payload_codec": DEFAULT_CODEC,
9060                        "value": envelope.clone(),
9061                    }),
9062                ),
9063            ],
9064            DEFAULT_CODEC,
9065        );
9066        assert_eq!(
9067            signal_context
9068                .wait_signal_avro_value("collision")
9069                .await
9070                .expect("typed signal collision arguments"),
9071            collision_values
9072        );
9073
9074        let child_context = workflow_context_with_codec(
9075            vec![
9076                history_event(
9077                    "ChildWorkflowScheduled",
9078                    json!({
9079                        "sequence": 1,
9080                        "child_workflow_instance_id": "collision-child",
9081                        "child_workflow_run_id": "collision-run",
9082                        "child_workflow_type": "collision.child",
9083                    }),
9084                ),
9085                history_event(
9086                    "ChildRunCompleted",
9087                    json!({
9088                        "sequence": 1,
9089                        "child_workflow_instance_id": "collision-child",
9090                        "child_workflow_run_id": "collision-run",
9091                        "child_workflow_type": "collision.child",
9092                        "payload_codec": DEFAULT_CODEC,
9093                        "result": envelope,
9094                    }),
9095                ),
9096            ],
9097            DEFAULT_CODEC,
9098        );
9099        let child = child_context
9100            .start_child_workflow_avro_value(
9101                "collision.child",
9102                ChildWorkflowOptions::new("collision-workers"),
9103                AvroValue::Array(Vec::new()),
9104            )
9105            .await
9106            .expect("typed child collision result");
9107        assert_eq!(child.result, expected);
9108    }
9109
9110    #[tokio::test]
9111    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9112        let client = Client::new("http://127.0.0.1:8080").expect("client");
9113        let mut worker = Worker::new(client, "rust-workers");
9114        worker.register_replayed_workflow_avro_value(
9115            "typed.replayed",
9116            || (),
9117            |_ctx, input, _state| async move { Ok(input) },
9118        );
9119        worker.register_replayed_query_avro_value::<(), _, _>(
9120            "typed.replayed",
9121            "inspect",
9122            |ctx, _state, args| async move {
9123                let mut signals = ctx.signals_avro_value("collision");
9124                let signal = signals
9125                    .pop()
9126                    .map(AvroValue::Array)
9127                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9128                Ok(AvroValue::Array(vec![
9129                    ctx.workflow_input_avro_value().clone(),
9130                    signal,
9131                    args,
9132                ]))
9133            },
9134        );
9135        let arguments = AvroValue::Array(projection_collision_probe());
9136        let signal_arguments =
9137            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9138        let task = QueryTask {
9139            query_task_id: "query-typed-replay".to_string(),
9140            query_task_attempt: 1,
9141            lease_owner: Some("rust-worker".to_string()),
9142            workflow_id: Some("typed-replay".to_string()),
9143            run_id: Some("run-typed-replay".to_string()),
9144            workflow_type: "typed.replayed".to_string(),
9145            query_name: "inspect".to_string(),
9146            payload_codec: DEFAULT_CODEC.to_string(),
9147            workflow_arguments: Some(
9148                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9149            ),
9150            query_arguments: Some(
9151                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9152            ),
9153            history_events: vec![history_event(
9154                "SignalReceived",
9155                json!({
9156                    "signal_id": "collision-signal",
9157                    "signal_name": "collision",
9158                    "workflow_sequence": 1,
9159                    "payload_codec": DEFAULT_CODEC,
9160                    "arguments": signal_arguments,
9161                }),
9162            )],
9163            history_export: None,
9164            run_status: Some("completed".to_string()),
9165        };
9166
9167        assert_eq!(
9168            worker
9169                .execute_query_task(task)
9170                .await
9171                .expect("typed replay query"),
9172            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9173        );
9174    }
9175
9176    #[test]
9177    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9178        let value = BTreeMap::from([(1_i32, "integer key")]);
9179        let error = PayloadEnvelope::avro(&value)
9180            .expect_err("integer map keys must fail")
9181            .to_string();
9182
9183        assert!(error.contains("invalid_map_key"));
9184    }
9185
9186    #[test]
9187    fn json_codec_remains_plain_json() {
9188        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9189        let envelope = PayloadEnvelope::json(&value).expect("encode");
9190
9191        assert_eq!(envelope.codec, JSON_CODEC);
9192        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9193        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9194    }
9195
9196    #[test]
9197    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9198        let envelope = PayloadEnvelope {
9199            codec: DEFAULT_CODEC.to_string(),
9200            blob: BASE64.encode([0x01]),
9201        };
9202
9203        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9204        assert!(error.to_string().contains("invalid_payload_framing"));
9205    }
9206
9207    #[test]
9208    fn workflow_context_schedules_activity_until_completion_is_in_history() {
9209        let ctx = WorkflowContext {
9210            state: Arc::new(Mutex::new(
9211                WorkflowState::new_with_identity(
9212                    Vec::new(),
9213                    Some("wf-parent".to_string()),
9214                    Some("run-parent".to_string()),
9215                    "rust-workers".to_string(),
9216                    DEFAULT_CODEC.to_string(),
9217                    None,
9218                )
9219                .expect("workflow state"),
9220            )),
9221        };
9222
9223        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9224        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9225        assert!(matches!(
9226            call.as_mut().poll(&mut task_context),
9227            Poll::Pending
9228        ));
9229
9230        let commands = ctx.take_commands().expect("commands");
9231        assert_eq!(commands[0]["type"], "schedule_activity");
9232        assert_eq!(commands[0]["activity_type"], "hello.activity");
9233    }
9234
9235    #[test]
9236    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9237        let ctx = workflow_context(Vec::new());
9238        let options = ActivityOptions::new()
9239            .task_queue("payments")
9240            .retry_policy(
9241                ActivityRetryPolicy::new(4)
9242                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9243                    .non_retryable_error_type("ValidationError"),
9244            )
9245            .start_to_close_timeout(Duration::from_secs(120))
9246            .schedule_to_start_timeout(Duration::from_secs(10))
9247            .schedule_to_close_timeout(Duration::from_secs(300))
9248            .heartbeat_timeout(Duration::from_secs(15));
9249        let mut call = Box::pin(ctx.activity_with_options(
9250            "charge-card",
9251            options,
9252            json!([{"order_id": "o-1"}]),
9253        ));
9254        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9255
9256        assert!(matches!(
9257            call.as_mut().poll(&mut task_context),
9258            Poll::Pending
9259        ));
9260        assert!(matches!(
9261            call.as_mut().poll(&mut task_context),
9262            Poll::Pending
9263        ));
9264
9265        let commands = ctx.take_commands().expect("activity command");
9266        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9267        assert_eq!(commands[0]["queue"], "payments");
9268        assert_eq!(
9269            commands[0]["retry_policy"],
9270            json!({
9271                "max_attempts": 4,
9272                "backoff_seconds": [1, 3, 9],
9273                "non_retryable_error_types": ["ValidationError"],
9274            })
9275        );
9276        assert_eq!(commands[0]["start_to_close_timeout"], 120);
9277        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9278        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9279        assert_eq!(commands[0]["heartbeat_timeout"], 15);
9280    }
9281
9282    #[test]
9283    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9284        let ctx = workflow_context(Vec::new());
9285        let options = ActivityOptions::new().retry_policy(
9286            ActivityRetryPolicy::new(3)
9287                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9288        );
9289        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9290        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9291
9292        assert!(matches!(
9293            call.as_mut().poll(&mut task_context),
9294            Poll::Pending
9295        ));
9296        assert_eq!(
9297            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9298            json!([1, 2])
9299        );
9300    }
9301
9302    #[test]
9303    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9304        let cases = [
9305            (
9306                ActivityOptions::new().task_queue("  "),
9307                ActivityOptionsErrorKind::EmptyTaskQueue,
9308            ),
9309            (
9310                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9311                ActivityOptionsErrorKind::EmptyRetryPolicy,
9312            ),
9313            (
9314                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9315                ActivityOptionsErrorKind::InvalidMaxAttempts,
9316            ),
9317            (
9318                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9319                    max_attempts: None,
9320                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9321                    non_retryable_error_types: Vec::new(),
9322                }),
9323                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9324            ),
9325            (
9326                ActivityOptions::new().retry_policy(
9327                    ActivityRetryPolicy::new(2)
9328                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9329                ),
9330                ActivityOptionsErrorKind::TooManyBackoffIntervals,
9331            ),
9332            (
9333                ActivityOptions::new().retry_policy(
9334                    ActivityRetryPolicy::new(2).exponential_backoff(
9335                        Duration::from_secs(1),
9336                        0,
9337                        None,
9338                    ),
9339                ),
9340                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9341            ),
9342            (
9343                ActivityOptions::new()
9344                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
9345                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9346            ),
9347            (
9348                ActivityOptions::new().retry_policy(
9349                    ActivityRetryPolicy::new(10_002).exponential_backoff(
9350                        Duration::from_secs(1),
9351                        1,
9352                        None,
9353                    ),
9354                ),
9355                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9356            ),
9357            (
9358                ActivityOptions::new().retry_policy(
9359                    ActivityRetryPolicy::new(2)
9360                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9361                ),
9362                ActivityOptionsErrorKind::BackoffOverflow,
9363            ),
9364        ];
9365
9366        for (options, expected_kind) in cases {
9367            let ctx = workflow_context(Vec::new());
9368            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9369            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9370            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9371                call.as_mut().poll(&mut task_context)
9372            else {
9373                panic!("expected typed activity validation error");
9374            };
9375            assert_eq!(error.kind, expected_kind);
9376            assert!(ctx.take_commands().expect("commands").is_empty());
9377        }
9378    }
9379
9380    #[test]
9381    fn activity_options_validate_positive_and_ordered_timeouts() {
9382        let zero_timeout_cases = [
9383            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9384            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9385            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9386            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9387        ];
9388        for options in zero_timeout_cases {
9389            assert_eq!(
9390                options.validate().expect_err("zero timeout").kind,
9391                ActivityOptionsErrorKind::TimeoutNotPositive
9392            );
9393        }
9394
9395        let ordering_cases = [
9396            ActivityOptions::new()
9397                .heartbeat_timeout(Duration::from_secs(11))
9398                .start_to_close_timeout(Duration::from_secs(10)),
9399            ActivityOptions::new()
9400                .start_to_close_timeout(Duration::from_secs(31))
9401                .schedule_to_close_timeout(Duration::from_secs(30)),
9402            ActivityOptions::new()
9403                .schedule_to_start_timeout(Duration::from_secs(31))
9404                .schedule_to_close_timeout(Duration::from_secs(30)),
9405        ];
9406        for options in ordering_cases {
9407            assert_eq!(
9408                options.validate().expect_err("timeout order").kind,
9409                ActivityOptionsErrorKind::TimeoutOrder
9410            );
9411        }
9412
9413        assert_eq!(
9414            ActivityOptions::new()
9415                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9416                .validate()
9417                .expect_err("protocol integer overflow")
9418                .kind,
9419            ActivityOptionsErrorKind::TimeoutOverflow
9420        );
9421    }
9422
9423    #[test]
9424    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9425        let ctx = workflow_context(completed_retry_activity_history());
9426        let mut call =
9427            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9428        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9429
9430        assert!(matches!(
9431            call.as_mut().poll(&mut task_context),
9432            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9433        ));
9434        assert!(ctx.take_commands().expect("commands").is_empty());
9435        ctx.ensure_history_consumed().expect("history consumed");
9436    }
9437
9438    #[test]
9439    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9440        let mut options = retry_activity_options();
9441        options
9442            .retry_policy
9443            .as_mut()
9444            .expect("retry policy")
9445            .non_retryable_error_types
9446            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9447
9448        let new_ctx = workflow_context(Vec::new());
9449        let mut new_call =
9450            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9451        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9452        assert!(matches!(
9453            new_call.as_mut().poll(&mut task_context),
9454            Poll::Pending
9455        ));
9456        let commands = new_ctx.take_commands().expect("commands");
9457        assert_eq!(commands.len(), 1);
9458        assert_eq!(
9459            commands[0]["retry_policy"]["non_retryable_error_types"],
9460            json!(["PermanentError"])
9461        );
9462
9463        let replay_ctx = workflow_context(completed_retry_activity_history());
9464        let mut replay_call =
9465            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9466        assert!(matches!(
9467            replay_call.as_mut().poll(&mut task_context),
9468            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9469        ));
9470        assert!(replay_ctx.take_commands().expect("commands").is_empty());
9471        replay_ctx
9472            .ensure_history_consumed()
9473            .expect("history consumed");
9474    }
9475
9476    #[test]
9477    fn replayed_intermediate_retry_remains_pending_across_restarts() {
9478        let history = completed_retry_activity_history()
9479            .into_iter()
9480            .take(3)
9481            .collect::<Vec<_>>();
9482
9483        for _restart in 0..2 {
9484            let ctx = workflow_context(history.clone());
9485            let mut call =
9486                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9487            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9488            assert!(matches!(
9489                call.as_mut().poll(&mut task_context),
9490                Poll::Pending
9491            ));
9492            assert!(ctx.take_commands().expect("commands").is_empty());
9493        }
9494    }
9495
9496    #[test]
9497    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9498        let mut changed_queue = retry_activity_options();
9499        changed_queue.task_queue = Some("different-queue".to_string());
9500
9501        let mut changed_max_attempts = retry_activity_options();
9502        let retry_policy = changed_max_attempts
9503            .retry_policy
9504            .as_mut()
9505            .expect("retry policy");
9506        retry_policy.max_attempts = Some(4);
9507
9508        let mut changed_backoff = retry_activity_options();
9509        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9510        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9511            Duration::from_secs(3),
9512            Duration::from_secs(4),
9513        ]));
9514
9515        let mut changed_non_retryable_types = retry_activity_options();
9516        let retry_policy = changed_non_retryable_types
9517            .retry_policy
9518            .as_mut()
9519            .expect("retry policy");
9520        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9521
9522        let mut changed_start_to_close = retry_activity_options();
9523        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9524        let mut changed_schedule_to_start = retry_activity_options();
9525        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9526        let mut changed_schedule_to_close = retry_activity_options();
9527        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9528        let mut changed_heartbeat = retry_activity_options();
9529        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9530
9531        let cases = [
9532            (changed_queue, "activity_task_queue_mismatch"),
9533            (changed_max_attempts, "activity_retry_policy_mismatch"),
9534            (changed_backoff, "activity_retry_policy_mismatch"),
9535            (
9536                changed_non_retryable_types,
9537                "activity_retry_policy_mismatch",
9538            ),
9539            (changed_start_to_close, "activity_retry_policy_mismatch"),
9540            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9541            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9542            (changed_heartbeat, "activity_retry_policy_mismatch"),
9543        ];
9544
9545        for (options, expected_reason) in cases {
9546            let ctx = workflow_context(completed_retry_activity_history());
9547            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9548            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9549            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9550                call.as_mut().poll(&mut task_context)
9551            else {
9552                panic!("changed activity options must fail replay");
9553            };
9554            assert_eq!(failure.reason, expected_reason);
9555            assert_eq!(failure.sequence, Some(1));
9556            assert!(ctx.take_commands().expect("commands").is_empty());
9557        }
9558    }
9559
9560    #[test]
9561    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9562        let cases = [
9563            (
9564                "execution_mode",
9565                json!("local"),
9566                "activity_execution_mode_mismatch",
9567            ),
9568            (
9569                "snapshot_version",
9570                json!(2),
9571                "activity_retry_policy_mismatch",
9572            ),
9573        ];
9574
9575        for (field, value, expected_reason) in cases {
9576            let mut history = completed_retry_activity_history();
9577            let activity = history[0].payload["activity"]
9578                .as_object_mut()
9579                .expect("activity snapshot");
9580            if field == "execution_mode" {
9581                activity.insert(field.to_string(), value);
9582            } else {
9583                activity["retry_policy"]
9584                    .as_object_mut()
9585                    .expect("retry snapshot")
9586                    .insert(field.to_string(), value);
9587            }
9588
9589            let ctx = workflow_context(history);
9590            let mut call =
9591                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9592            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9593            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9594                call.as_mut().poll(&mut task_context)
9595            else {
9596                panic!("changed {field} must fail replay");
9597            };
9598            assert_eq!(failure.reason, expected_reason);
9599            assert_eq!(failure.sequence, Some(1));
9600            assert!(ctx.take_commands().expect("commands").is_empty());
9601        }
9602    }
9603
9604    #[test]
9605    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9606        let mut history = completed_retry_activity_history();
9607        let activity = history[0].payload["activity"]
9608            .as_object_mut()
9609            .expect("activity snapshot");
9610        activity.remove("execution_mode");
9611        activity.remove("retry_policy");
9612
9613        let mut current = retry_activity_options();
9614        current.start_to_close_timeout = Some(Duration::from_secs(45));
9615        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9616        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9617        current.heartbeat_timeout = Some(Duration::from_secs(12));
9618
9619        let ctx = workflow_context(history);
9620        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9621        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9622        assert!(matches!(
9623            call.as_mut().poll(&mut task_context),
9624            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9625        ));
9626        assert!(ctx.take_commands().expect("commands").is_empty());
9627        ctx.ensure_history_consumed().expect("history consumed");
9628    }
9629
9630    #[test]
9631    fn terminal_activity_failed_after_start_returns_typed_failure() {
9632        let history = vec![
9633            history_event(
9634                "ActivityScheduled",
9635                json!({
9636                    "sequence": 1,
9637                    "activity_type": "flaky",
9638                    "activity_execution_id": "act-terminal",
9639                    "activity": {
9640                        "id": "act-terminal",
9641                        "sequence": 1,
9642                        "type": "flaky",
9643                        "queue": "critical-activities",
9644                        "retry_policy": {
9645                            "snapshot_version": 1,
9646                            "max_attempts": 3,
9647                            "backoff_seconds": [2, 4],
9648                            "non_retryable_error_types": ["PermanentError"]
9649                        }
9650                    }
9651                }),
9652            ),
9653            history_event(
9654                "ActivityStarted",
9655                json!({
9656                    "sequence": 1,
9657                    "activity_type": "flaky",
9658                    "activity_execution_id": "act-terminal",
9659                    "activity_attempt_id": "attempt-1",
9660                    "attempt_number": 1
9661                }),
9662            ),
9663            history_event(
9664                "ActivityFailed",
9665                json!({
9666                    "sequence": 1,
9667                    "activity_type": "flaky",
9668                    "activity_execution_id": "act-terminal",
9669                    "activity_attempt_id": "attempt-1",
9670                    "attempt_number": 1,
9671                    "failure_id": "failure-terminal",
9672                    "failure_category": "activity",
9673                    "exception_type": "PermanentError",
9674                    "message": "cannot retry",
9675                    "non_retryable": true
9676                }),
9677            ),
9678        ];
9679        let ctx = workflow_context(history);
9680        let mut call =
9681            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9682        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9683
9684        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9685            call.as_mut().poll(&mut task_context)
9686        else {
9687            panic!("terminal ActivityFailed must settle the activity future");
9688        };
9689        assert_eq!(failure.kind, ActivityFailureKind::Failed);
9690        assert_eq!(
9691            failure.activity_execution_id.as_deref(),
9692            Some("act-terminal")
9693        );
9694        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9695        assert!(failure.non_retryable);
9696        assert!(ctx.take_commands().expect("commands").is_empty());
9697        ctx.ensure_history_consumed().expect("history consumed");
9698    }
9699
9700    #[test]
9701    fn activity_terminal_events_return_machine_readable_failures() {
9702        let cases = [
9703            (
9704                "ActivityFailed",
9705                json!({
9706                    "sequence": 1,
9707                    "activity_type": "charge-card",
9708                    "activity_execution_id": "act-1",
9709                    "activity_attempt_id": "attempt-2",
9710                    "attempt_number": 2,
9711                    "failure_id": "failure-1",
9712                    "failure_category": "activity",
9713                    "exception_type": "PaymentDeclined",
9714                    "exception_class": "payments.PaymentDeclined",
9715                    "message": "card declined",
9716                    "non_retryable": true
9717                }),
9718                ActivityFailureKind::Failed,
9719                "activity",
9720            ),
9721            (
9722                "ActivityCancelled",
9723                json!({
9724                    "sequence": 1,
9725                    "activity_type": "charge-card",
9726                    "activity_execution_id": "act-1",
9727                    "activity_attempt_id": "attempt-1"
9728                }),
9729                ActivityFailureKind::Cancelled,
9730                "cancelled",
9731            ),
9732        ];
9733
9734        for (event_type, payload, expected_kind, expected_reason) in cases {
9735            let ctx = workflow_context(vec![history_event(event_type, payload)]);
9736            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9737            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9738            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9739                call.as_mut().poll(&mut task_context)
9740            else {
9741                panic!("expected terminal activity failure");
9742            };
9743            assert_eq!(failure.kind, expected_kind);
9744            assert_eq!(failure.reason, expected_reason);
9745            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9746            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9747        }
9748    }
9749
9750    #[test]
9751    fn every_activity_timeout_class_is_typed() {
9752        for timeout_kind in [
9753            "start_to_close",
9754            "schedule_to_start",
9755            "schedule_to_close",
9756            "heartbeat",
9757        ] {
9758            let ctx = workflow_context(vec![history_event(
9759                "ActivityTimedOut",
9760                json!({
9761                    "sequence": 1,
9762                    "activity_type": "slow",
9763                    "activity_execution_id": "act-timeout",
9764                    "activity_attempt_id": "attempt-timeout",
9765                    "failure_category": "timeout",
9766                    "timeout_kind": timeout_kind,
9767                    "message": "deadline expired"
9768                }),
9769            )]);
9770            let mut call = Box::pin(ctx.activity("slow", json!([])));
9771            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9772            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9773                call.as_mut().poll(&mut task_context)
9774            else {
9775                panic!("expected timeout failure");
9776            };
9777            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9778            assert_eq!(failure.reason, timeout_kind);
9779            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9780            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9781        }
9782    }
9783
9784    #[test]
9785    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9786        let ctx = workflow_context(Vec::new());
9787        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9788        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9789
9790        assert!(matches!(
9791            sleep.as_mut().poll(&mut task_context),
9792            Poll::Pending
9793        ));
9794        assert!(matches!(
9795            sleep.as_mut().poll(&mut task_context),
9796            Poll::Pending
9797        ));
9798
9799        let commands = ctx.take_commands().expect("timer command");
9800        assert_eq!(
9801            commands,
9802            vec![json!({
9803                "type": "start_timer",
9804                "delay_seconds": 2,
9805            })]
9806        );
9807    }
9808
9809    #[test]
9810    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9811        let history = vec![
9812            history_event(
9813                "TimerScheduled",
9814                json!({
9815                    "sequence": 1,
9816                    "timer_id": "timer-1",
9817                    "delay_seconds": 5,
9818                    "fire_at": "2026-07-11T12:00:05Z",
9819                }),
9820            ),
9821            history_event(
9822                "TimerFired",
9823                json!({
9824                    "sequence": 1,
9825                    "timer_id": "timer-1",
9826                    "delay_seconds": 5,
9827                    "fire_at": "2026-07-11T12:00:05Z",
9828                    "fired_at": "2026-07-11T12:00:05Z",
9829                }),
9830            ),
9831        ];
9832
9833        for _restart in 0..2 {
9834            let ctx = workflow_context(history.clone());
9835            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9836            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9837            assert!(matches!(
9838                sleep.as_mut().poll(&mut task_context),
9839                Poll::Ready(Ok(()))
9840            ));
9841            assert!(ctx.take_commands().expect("commands").is_empty());
9842            ctx.ensure_history_consumed().expect("history consumed");
9843        }
9844    }
9845
9846    #[test]
9847    fn workflow_sleep_rejects_changed_delay_during_replay() {
9848        let ctx = workflow_context(vec![
9849            history_event(
9850                "TimerScheduled",
9851                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9852            ),
9853            history_event(
9854                "TimerFired",
9855                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9856            ),
9857        ]);
9858        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
9859        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9860
9861        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9862            sleep.as_mut().poll(&mut task_context)
9863        else {
9864            panic!("changed timer delay must be rejected");
9865        };
9866        assert_eq!(failure.reason, "timer_delay_mismatch");
9867        assert_eq!(failure.sequence, Some(1));
9868    }
9869
9870    #[test]
9871    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
9872        let lone_fire = WorkflowState::new(
9873            vec![history_event(
9874                "TimerFired",
9875                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9876            )],
9877            "rust-workers".to_string(),
9878            JSON_CODEC.to_string(),
9879            None,
9880        )
9881        .expect_err("TimerFired requires TimerScheduled");
9882        assert!(matches!(
9883            lone_fire,
9884            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9885                if reason == "timer_schedule_missing_or_duplicate"
9886        ));
9887
9888        let wrong_identity = WorkflowState::new(
9889            vec![
9890                history_event(
9891                    "TimerScheduled",
9892                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9893                ),
9894                history_event(
9895                    "TimerFired",
9896                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
9897                ),
9898            ],
9899            "rust-workers".to_string(),
9900            JSON_CODEC.to_string(),
9901            None,
9902        )
9903        .expect_err("fire must match scheduled timer identity");
9904        assert!(matches!(
9905            wrong_identity,
9906            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9907                if reason == "timer_identity_mismatch"
9908        ));
9909
9910        let duplicate_fire = WorkflowState::new(
9911            vec![
9912                history_event(
9913                    "TimerScheduled",
9914                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9915                ),
9916                history_event(
9917                    "TimerFired",
9918                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9919                ),
9920                history_event(
9921                    "TimerFired",
9922                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9923                ),
9924            ],
9925            "rust-workers".to_string(),
9926            JSON_CODEC.to_string(),
9927            None,
9928        )
9929        .expect_err("a durable timer cannot fire twice");
9930        assert!(matches!(
9931            duplicate_fire,
9932            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9933                if reason == "duplicate_timer_fire"
9934        ));
9935
9936        let wrong_fired_delay = WorkflowState::new(
9937            vec![
9938                history_event(
9939                    "TimerScheduled",
9940                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9941                ),
9942                history_event(
9943                    "TimerFired",
9944                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
9945                ),
9946            ],
9947            "rust-workers".to_string(),
9948            JSON_CODEC.to_string(),
9949            None,
9950        )
9951        .expect_err("timer schedule and fire delays must agree");
9952        assert!(matches!(
9953            wrong_fired_delay,
9954            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9955                if reason == "timer_history_delay_mismatch"
9956        ));
9957    }
9958
9959    #[test]
9960    fn replay_rejects_activity_moved_before_recorded_timer() {
9961        let ctx = workflow_context(vec![
9962            history_event(
9963                "TimerScheduled",
9964                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9965            ),
9966            history_event(
9967                "TimerFired",
9968                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9969            ),
9970            history_event(
9971                "ActivityCompleted",
9972                json!({
9973                    "sequence": 2,
9974                    "activity_type": "after-timer",
9975                    "payload_codec": "json",
9976                    "result": {"codec": "json", "blob": "\"done\""},
9977                }),
9978            ),
9979        ]);
9980        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
9981        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9982
9983        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9984            activity.as_mut().poll(&mut task_context)
9985        else {
9986            panic!("reordered durable command must be rejected");
9987        };
9988        assert_eq!(failure.reason, "recorded_command_mismatch");
9989        assert_eq!(failure.sequence, Some(1));
9990        assert_eq!(failure.expected.as_deref(), Some("timer"));
9991        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
9992    }
9993
9994    #[test]
9995    fn workflow_context_emits_a_typed_named_signal_wait() {
9996        let ctx = workflow_context(Vec::new());
9997        let mut signal = Box::pin(ctx.wait_signal("finish"));
9998        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9999
10000        assert!(matches!(
10001            signal.as_mut().poll(&mut task_context),
10002            Poll::Pending
10003        ));
10004        assert_eq!(
10005            ctx.take_commands().expect("signal-wait command"),
10006            vec![json!({
10007                "type": "open_signal_wait",
10008                "signal_name": "finish",
10009            })]
10010        );
10011    }
10012
10013    #[test]
10014    fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10015        let ctx = workflow_context(vec![
10016            history_event(
10017                "ConditionWaitOpened",
10018                json!({"sequence": 1, "condition_key": "signal:finish"}),
10019            ),
10020            history_event(
10021                "ConditionWaitSatisfied",
10022                json!({"sequence": 1, "condition_key": "signal:finish"}),
10023            ),
10024            history_event(
10025                "SignalReceived",
10026                json!({"signal_name": "finish", "arguments": []}),
10027            ),
10028        ]);
10029        let mut signal = Box::pin(ctx.wait_signal("finish"));
10030        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10031
10032        assert!(matches!(
10033            signal.as_mut().poll(&mut task_context),
10034            Poll::Pending
10035        ));
10036        assert_eq!(
10037            ctx.take_commands().expect("typed signal-wait command"),
10038            vec![json!({
10039                "type": "open_signal_wait",
10040                "signal_name": "finish",
10041            })]
10042        );
10043    }
10044
10045    #[test]
10046    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10047        let signal_then_timer = vec![
10048            history_event(
10049                "SignalWaitOpened",
10050                json!({"sequence": 1, "signal_name": "go"}),
10051            ),
10052            history_event(
10053                "SignalApplied",
10054                json!({
10055                    "sequence": 1,
10056                    "signal_name": "go",
10057                    "value": {"codec": "json", "blob": "[\"now\"]"},
10058                }),
10059            ),
10060            history_event(
10061                "TimerScheduled",
10062                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10063            ),
10064            history_event(
10065                "TimerFired",
10066                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10067            ),
10068        ];
10069
10070        let ctx = workflow_context(signal_then_timer.clone());
10071        let mut signal = Box::pin(ctx.wait_signal("go"));
10072        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10073        assert!(matches!(
10074            signal.as_mut().poll(&mut task_context),
10075            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10076        ));
10077        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10078        assert!(matches!(
10079            timer.as_mut().poll(&mut task_context),
10080            Poll::Ready(Ok(()))
10081        ));
10082        ctx.ensure_history_consumed()
10083            .expect("signal and timer history consumed in order");
10084
10085        let reordered = workflow_context(signal_then_timer);
10086        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10087        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10088            timer_first.as_mut().poll(&mut task_context)
10089        else {
10090            panic!("timer cannot consume signal-wait-first history");
10091        };
10092        assert_eq!(failure.reason, "recorded_command_mismatch");
10093        assert_eq!(failure.sequence, Some(1));
10094        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10095
10096        let timer_then_signal = vec![
10097            history_event(
10098                "TimerScheduled",
10099                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10100            ),
10101            history_event(
10102                "TimerFired",
10103                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10104            ),
10105            history_event(
10106                "SignalWaitOpened",
10107                json!({"sequence": 2, "signal_name": "go"}),
10108            ),
10109            history_event(
10110                "SignalApplied",
10111                json!({
10112                    "sequence": 2,
10113                    "signal_name": "go",
10114                    "value": {"codec": "json", "blob": "[]"},
10115                }),
10116            ),
10117        ];
10118        let reordered = workflow_context(timer_then_signal);
10119        let mut signal_first = Box::pin(reordered.wait_signal("go"));
10120        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10121            signal_first.as_mut().poll(&mut task_context)
10122        else {
10123            panic!("signal wait cannot consume timer-first history");
10124        };
10125        assert_eq!(failure.reason, "recorded_command_mismatch");
10126        assert_eq!(failure.sequence, Some(1));
10127        assert_eq!(failure.expected.as_deref(), Some("timer"));
10128    }
10129
10130    #[test]
10131    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10132        let duplicate_timer = WorkflowState::new(
10133            vec![
10134                history_event(
10135                    "TimerScheduled",
10136                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10137                ),
10138                history_event(
10139                    "TimerScheduled",
10140                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10141                ),
10142            ],
10143            "rust-workers".to_string(),
10144            JSON_CODEC.to_string(),
10145            None,
10146        )
10147        .expect_err("one workflow sequence cannot schedule two timers");
10148        assert!(matches!(
10149            duplicate_timer,
10150            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10151                if reason == "timer_schedule_missing_or_duplicate"
10152        ));
10153
10154        let colliding_kinds = WorkflowState::new(
10155            vec![
10156                history_event(
10157                    "TimerScheduled",
10158                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10159                ),
10160                history_event(
10161                    "ActivityCompleted",
10162                    json!({"sequence": 1, "activity_type": "same-sequence"}),
10163                ),
10164            ],
10165            "rust-workers".to_string(),
10166            JSON_CODEC.to_string(),
10167            None,
10168        )
10169        .expect_err("one workflow sequence cannot identify two command kinds");
10170        assert!(matches!(
10171            colliding_kinds,
10172            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10173                if reason == "durable_command_sequence_collision"
10174        ));
10175
10176        let duplicate_signal_wait = WorkflowState::new(
10177            vec![
10178                history_event(
10179                    "SignalWaitOpened",
10180                    json!({"sequence": 1, "signal_name": "go"}),
10181                ),
10182                history_event(
10183                    "SignalWaitOpened",
10184                    json!({"sequence": 1, "signal_name": "go"}),
10185                ),
10186            ],
10187            "rust-workers".to_string(),
10188            JSON_CODEC.to_string(),
10189            None,
10190        )
10191        .expect_err("one workflow sequence cannot open two signal waits");
10192        assert!(matches!(
10193            duplicate_signal_wait,
10194            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10195                if reason == "signal_wait_open_missing_or_duplicate"
10196        ));
10197    }
10198
10199    #[test]
10200    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10201        let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10202            .expect("side-effect result");
10203        let ctx = workflow_context(vec![history_event(
10204            "SideEffectRecorded",
10205            json!({"sequence": 99, "result": result}),
10206        )]);
10207
10208        let replayed: Value = ctx
10209            .side_effect(|| panic!("recorded side effect must not run"))
10210            .expect("positive global workflow sequence is valid");
10211        assert_eq!(replayed, json!({"captured": true}));
10212        ctx.ensure_history_consumed().expect("history consumed");
10213    }
10214
10215    #[test]
10216    fn workflow_history_rejects_zero_and_descending_command_sequences() {
10217        let result =
10218            encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10219        let zero = WorkflowState::new(
10220            vec![history_event(
10221                "SideEffectRecorded",
10222                json!({"sequence": 0, "result": result.clone()}),
10223            )],
10224            "rust-workers".to_string(),
10225            JSON_CODEC.to_string(),
10226            None,
10227        )
10228        .expect_err("durable command sequences must be positive");
10229        assert!(matches!(
10230            zero,
10231            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10232                if reason == "durable_command_sequence_invalid"
10233        ));
10234
10235        let descending = WorkflowState::new(
10236            vec![
10237                history_event(
10238                    "SideEffectRecorded",
10239                    json!({"sequence": 3, "result": result}),
10240                ),
10241                history_event(
10242                    "VersionMarkerRecorded",
10243                    json!({
10244                        "sequence": 2,
10245                        "change_id": "descending-marker",
10246                        "version": 1,
10247                        "min_supported": 1,
10248                        "max_supported": 1,
10249                    }),
10250                ),
10251            ],
10252            "rust-workers".to_string(),
10253            JSON_CODEC.to_string(),
10254            None,
10255        )
10256        .expect_err("new durable commands must remain strictly ordered");
10257        let Error::NonDeterministicReplay(failure) = descending else {
10258            panic!("expected typed replay failure");
10259        };
10260        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10261        assert_eq!(failure.sequence, Some(2));
10262        assert_eq!(
10263            failure.expected.as_deref(),
10264            Some("workflow sequence greater than 3")
10265        );
10266        assert_eq!(failure.actual.as_deref(), Some("2"));
10267    }
10268
10269    #[test]
10270    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10271        fn worker() -> Worker {
10272            let client = Client::new("http://127.0.0.1:8080").expect("client");
10273            let mut worker = Worker::new(client, "rust-workers");
10274            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10275                ctx.wait_signal("finish").await?;
10276                let marker: String =
10277                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10278                assert_eq!(marker, "after-finish");
10279                Ok(json!("finished"))
10280            });
10281            worker
10282        }
10283
10284        let marker =
10285            encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10286        let task = workflow_task(
10287            "rust.finish-after-gaps",
10288            vec![
10289                history_event(
10290                    "SignalWaitOpened",
10291                    json!({"sequence": 1, "signal_name": "finish"}),
10292                ),
10293                history_event(
10294                    "SignalReceived",
10295                    json!({
10296                        "signal_id": "increment-3",
10297                        "signal_name": "increment",
10298                        "workflow_sequence": 2,
10299                        "payload_codec": "json",
10300                        "arguments": {"codec": "json", "blob": "[3]"},
10301                    }),
10302                ),
10303                history_event(
10304                    "SignalReceived",
10305                    json!({
10306                        "signal_id": "increment-5",
10307                        "signal_name": "increment",
10308                        "workflow_sequence": 3,
10309                        "payload_codec": "json",
10310                        "arguments": {"codec": "json", "blob": "[5]"},
10311                    }),
10312                ),
10313                history_event(
10314                    "SignalReceived",
10315                    json!({
10316                        "signal_id": "finish",
10317                        "signal_name": "finish",
10318                        "workflow_sequence": 4,
10319                        "payload_codec": "json",
10320                        "arguments": {"codec": "json", "blob": "[]"},
10321                    }),
10322                ),
10323                history_event(
10324                    "SignalApplied",
10325                    json!({
10326                        "sequence": 1,
10327                        "signal_id": "finish",
10328                        "signal_name": "finish",
10329                        "payload_codec": "json",
10330                        "value": {"codec": "json", "blob": "[]"},
10331                    }),
10332                ),
10333                history_event(
10334                    "SideEffectRecorded",
10335                    json!({"sequence": 5, "result": marker}),
10336                ),
10337            ],
10338            JSON_CODEC,
10339        );
10340
10341        for _original_or_cold_worker in 0..2 {
10342            let commands = worker()
10343                .execute_workflow_task(task.clone())
10344                .expect("signal gaps preserve deterministic replay");
10345            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10346            assert_eq!(commands[0]["type"], "complete_workflow");
10347            assert_eq!(
10348                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10349                json!("finished")
10350            );
10351        }
10352    }
10353
10354    #[test]
10355    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10356        let ctx = workflow_context(Vec::new());
10357        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10358        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10359        assert!(matches!(
10360            sleep.as_mut().poll(&mut task_context),
10361            Poll::Ready(Err(Error::TimerDurationOverflow))
10362        ));
10363        assert!(ctx.take_commands().expect("commands").is_empty());
10364    }
10365
10366    #[test]
10367    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10368        let client = Client::new("http://127.0.0.1:8080").expect("client");
10369        let mut worker = Worker::new(client, "rust-workers");
10370        worker.register_workflow("rust.timer", |ctx, _input| async move {
10371            ctx.sleep(Duration::from_secs(5)).await?;
10372            ctx.activity("after-timer", json!([])).await
10373        });
10374
10375        let task = |history_events| WorkflowTask {
10376            task_id: "wft-rust-timer-1".to_string(),
10377            workflow_id: Some("wf-rust-timer".to_string()),
10378            run_id: Some("run-rust-timer".to_string()),
10379            workflow_type: "rust.timer".to_string(),
10380            payload_codec: JSON_CODEC.to_string(),
10381            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10382            history_events,
10383            total_history_events: None,
10384            history_size_bytes: None,
10385            continue_as_new_recommended: None,
10386            history_budget_pressure: None,
10387            next_history_page_token: None,
10388            workflow_task_attempt: 1,
10389            workflow_signal_id: None,
10390            signal_name: None,
10391            signal_arguments: None,
10392            workflow_update_id: None,
10393            update_name: None,
10394            lease_owner: Some("rust-worker".to_string()),
10395        };
10396
10397        let initial = worker
10398            .execute_workflow_task(task(Vec::new()))
10399            .expect("initial timer task");
10400        assert_eq!(
10401            initial,
10402            vec![json!({"type": "start_timer", "delay_seconds": 5})]
10403        );
10404
10405        let replayed = worker
10406            .execute_workflow_task(task(vec![
10407                history_event(
10408                    "TimerScheduled",
10409                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10410                ),
10411                history_event(
10412                    "TimerFired",
10413                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10414                ),
10415                history_event(
10416                    "ActivityCompleted",
10417                    json!({
10418                        "sequence": 2,
10419                        "activity_type": "after-timer",
10420                        "payload_codec": "json",
10421                        "result": {"codec": "json", "blob": "\"done\""},
10422                    }),
10423                ),
10424            ]))
10425            .expect("replayed workflow task");
10426        assert_eq!(replayed.len(), 1);
10427        assert_eq!(replayed[0]["type"], "complete_workflow");
10428        assert_eq!(
10429            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10430            json!("done")
10431        );
10432    }
10433
10434    #[test]
10435    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10436        let client = Client::new("http://127.0.0.1:8080").expect("client");
10437        let mut worker = Worker::new(client, "rust-workers");
10438        worker.register_workflow("rust.continue", |ctx, _input| async move {
10439            ctx.continue_as_new_with_options(
10440                ContinueAsNewOptions::new()
10441                    .workflow_type("rust.next")
10442                    .task_queue("next-workers"),
10443                json!([2, {"cursor": "next"}]),
10444            )
10445        });
10446
10447        let commands = worker
10448            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10449            .expect("continue-as-new command");
10450
10451        assert_eq!(commands.len(), 1);
10452        assert_eq!(commands[0]["type"], "continue_as_new");
10453        assert_eq!(commands[0]["workflow_type"], "rust.next");
10454        assert_eq!(commands[0]["queue"], "next-workers");
10455        assert_eq!(
10456            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10457                .expect("continue-as-new arguments"),
10458            json!([2, {"cursor": "next"}])
10459        );
10460    }
10461
10462    #[test]
10463    fn continue_as_new_preserves_typed_arguments() {
10464        let client = Client::new("http://127.0.0.1:8080").expect("client");
10465        let mut worker = Worker::new(client, "rust-workers");
10466        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10467            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10468            unreachable!("continue-as-new returns a control-flow error")
10469        });
10470
10471        let commands = worker
10472            .execute_workflow_task(workflow_task(
10473                "rust.typed-continue",
10474                Vec::new(),
10475                DEFAULT_CODEC,
10476            ))
10477            .expect("typed continue-as-new command");
10478
10479        assert_eq!(commands[0]["type"], "continue_as_new");
10480        assert_eq!(
10481            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10482                .expect("typed continue arguments"),
10483            AvroValue::Array(vec![typed_fidelity_probe()])
10484        );
10485    }
10486
10487    #[test]
10488    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10489        let client = Client::new("http://127.0.0.1:8080").expect("client");
10490        let mut worker = Worker::new(client, "rust-workers");
10491        worker.register_workflow("rust.continue", |ctx, _input| async move {
10492            ctx.continue_as_new(json!([2]))
10493        });
10494        let task = workflow_task(
10495            "rust.continue",
10496            vec![history_event(
10497                "WorkflowContinuedAsNew",
10498                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10499            )],
10500            JSON_CODEC,
10501        );
10502
10503        for _worker_restart_or_redelivery in 0..2 {
10504            let commands = worker
10505                .execute_workflow_task(task.clone())
10506                .expect("recorded transition replays");
10507            assert!(
10508                commands.is_empty(),
10509                "replay must not emit another successor"
10510            );
10511        }
10512    }
10513
10514    #[test]
10515    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10516        let ctx = workflow_context(Vec::new());
10517        let error = ctx
10518            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
10519            .expect_err("blank queue must be rejected");
10520
10521        let Error::InvalidContinueAsNewOptions(error) = error else {
10522            panic!("expected typed continue-as-new validation error");
10523        };
10524        assert_eq!(error.field, "task_queue");
10525        assert!(ctx.take_commands().expect("commands").is_empty());
10526    }
10527
10528    #[test]
10529    fn workflow_context_exposes_server_history_budget() {
10530        let client = Client::new("http://127.0.0.1:8080").expect("client");
10531        let mut worker = Worker::new(client, "rust-workers");
10532        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10533            let budget = ctx.history_budget()?;
10534            Ok(json!({
10535                "events": budget.event_count,
10536                "bytes": budget.size_bytes,
10537                "recommended": budget.continue_as_new_recommended,
10538                "pressure": budget.pressure,
10539            }))
10540        });
10541        let task: WorkflowTask = serde_json::from_value(json!({
10542            "task_id": "task-history-budget",
10543            "workflow_type": "rust.history-budget",
10544            "payload_codec": JSON_CODEC,
10545            "history_events": [],
10546            "total_history_events": 480,
10547            "history_size_bytes": 1_048_576,
10548            "continue_as_new_recommended": true,
10549            "history_budget_pressure": "continue_as_new_recommended",
10550        }))
10551        .expect("published workflow task");
10552
10553        let commands = worker
10554            .execute_workflow_task(task)
10555            .expect("history-budget workflow");
10556        let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10557        assert_eq!(result["events"], 480);
10558        assert_eq!(result["bytes"], 1_048_576);
10559        assert_eq!(result["recommended"], true);
10560        assert_eq!(result["pressure"], "continue_as_new_recommended");
10561    }
10562
10563    #[test]
10564    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10565        let client = Client::new("http://127.0.0.1:8080").expect("client");
10566        let mut worker = Worker::new(client, "rust-workers");
10567        worker.register_workflow("rust.failing", |_ctx, _input| async move {
10568            Err(Error::Codec("rust_conformance_failure".to_string()))
10569        });
10570        let task = WorkflowTask {
10571            task_id: "wft-rust-failing-1".to_string(),
10572            workflow_id: Some("wf-rust-failing".to_string()),
10573            run_id: Some("run-rust-failing".to_string()),
10574            workflow_type: "rust.failing".to_string(),
10575            payload_codec: JSON_CODEC.to_string(),
10576            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10577            history_events: Vec::new(),
10578            total_history_events: Some(0),
10579            history_size_bytes: None,
10580            continue_as_new_recommended: None,
10581            history_budget_pressure: None,
10582            next_history_page_token: None,
10583            workflow_task_attempt: 1,
10584            workflow_signal_id: None,
10585            signal_name: None,
10586            signal_arguments: None,
10587            workflow_update_id: None,
10588            update_name: None,
10589            lease_owner: Some("rust-worker".to_string()),
10590        };
10591
10592        let commands = worker
10593            .execute_workflow_task(task)
10594            .expect("handler failure becomes a workflow command");
10595
10596        assert_eq!(commands.len(), 1);
10597        assert_eq!(commands[0]["type"], "fail_workflow");
10598        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10599        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10600        assert_eq!(commands[0]["non_retryable"], false);
10601        assert_eq!(
10602            commands[0]["message"],
10603            "codec error: rust_conformance_failure"
10604        );
10605        assert_eq!(
10606            commands[0]["exception"]["message"],
10607            "codec error: rust_conformance_failure"
10608        );
10609    }
10610
10611    #[test]
10612    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10613        let client = Client::new("http://127.0.0.1:8080").expect("client");
10614        let mut worker = Worker::new(client, "rust-workers");
10615        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10616            let _: String = ctx.side_effect(|| "captured".to_string())?;
10617            Err(Error::WorkerLoop("application failure".to_string()))
10618        });
10619
10620        let commands = worker
10621            .execute_workflow_task(workflow_task(
10622                "rust.failing-after-side-effect",
10623                Vec::new(),
10624                JSON_CODEC,
10625            ))
10626            .expect("ordinary failure remains a workflow decision");
10627
10628        assert_eq!(commands.len(), 2);
10629        assert_eq!(commands[0]["type"], "record_side_effect");
10630        assert_eq!(commands[1]["type"], "fail_workflow");
10631    }
10632
10633    #[test]
10634    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10635        let client = Client::new("http://127.0.0.1:8080").expect("client");
10636        let mut worker = Worker::new(client, "rust-workers");
10637        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10638            Err(Error::WorkerLoop("application failure".to_string()))
10639        });
10640        let result =
10641            encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10642
10643        let error = worker
10644            .execute_workflow_task(workflow_task(
10645                "rust.removed-side-effect",
10646                vec![history_event(
10647                    "SideEffectRecorded",
10648                    json!({"sequence": 1, "result": result}),
10649                )],
10650                JSON_CODEC,
10651            ))
10652            .expect_err("removed committed history must not become fail_workflow");
10653
10654        let Error::NonDeterministicReplay(failure) = error else {
10655            panic!("expected typed replay failure");
10656        };
10657        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10658        assert_eq!(failure.sequence, Some(1));
10659        assert_eq!(failure.expected.as_deref(), Some("side effect"));
10660    }
10661
10662    #[test]
10663    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10664        let client = Client::new("http://127.0.0.1:8080").expect("client");
10665        let mut worker = Worker::new(client, "rust-workers");
10666        worker.register_workflow(
10667            "rust.side-effect-before-marker-error",
10668            |ctx, _input| async move {
10669                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10670                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10671                ctx.get_version("restart-safe", 2, 2)?;
10672                Ok(Value::Null)
10673            },
10674        );
10675
10676        let error = worker
10677            .execute_workflow_task(workflow_task(
10678                "rust.side-effect-before-marker-error",
10679                vec![history_event(
10680                    "VersionMarkerRecorded",
10681                    json!({
10682                        "sequence": 1,
10683                        "change_id": "restart-safe",
10684                        "version": 1,
10685                        "min_supported": 1,
10686                        "max_supported": 1,
10687                    }),
10688                )],
10689                JSON_CODEC,
10690            ))
10691            .expect_err("replay error must return no queued workflow commands");
10692
10693        let Error::NonDeterministicReplay(failure) = error else {
10694            panic!("expected typed replay failure");
10695        };
10696        assert_eq!(failure.reason, "version_marker_incompatible_range");
10697        assert_eq!(failure.sequence, Some(1));
10698    }
10699
10700    #[test]
10701    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10702        let client = Client::new("http://127.0.0.1:8080").expect("client");
10703        let mut worker = Worker::new(client, "rust-workers");
10704        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10705            ctx.sleep(Duration::from_secs(5)).await?;
10706            Ok(json!({"status": "timer fired"}))
10707        });
10708
10709        let task = WorkflowTask {
10710            task_id: "wft-rust-timer-pending".to_string(),
10711            workflow_id: Some("wf-rust-timer".to_string()),
10712            run_id: Some("run-rust-timer".to_string()),
10713            workflow_type: "rust.timer.pending".to_string(),
10714            payload_codec: JSON_CODEC.to_string(),
10715            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10716            history_events: vec![history_event(
10717                "TimerScheduled",
10718                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10719            )],
10720            total_history_events: Some(1),
10721            history_size_bytes: None,
10722            continue_as_new_recommended: None,
10723            history_budget_pressure: None,
10724            next_history_page_token: None,
10725            workflow_task_attempt: 1,
10726            workflow_signal_id: None,
10727            signal_name: None,
10728            signal_arguments: None,
10729            workflow_update_id: None,
10730            update_name: None,
10731            lease_owner: Some("rust-worker".to_string()),
10732        };
10733
10734        for _redelivery_or_restart in 0..2 {
10735            let commands = worker
10736                .execute_workflow_task(task.clone())
10737                .expect("recorded timer remains pending");
10738            assert!(
10739                commands.is_empty(),
10740                "recorded timer must not be rescheduled"
10741            );
10742        }
10743    }
10744
10745    #[test]
10746    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10747        let client = Client::new("http://127.0.0.1:8080").expect("client");
10748        let mut worker = Worker::new(client, "rust-workers");
10749        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10750            Ok(json!({"status": "completed"}))
10751        });
10752        let task = WorkflowTask {
10753            task_id: "wft-rust-timer-removed".to_string(),
10754            workflow_id: Some("wf-rust-timer".to_string()),
10755            run_id: Some("run-rust-timer".to_string()),
10756            workflow_type: "rust.timer.removed".to_string(),
10757            payload_codec: JSON_CODEC.to_string(),
10758            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10759            history_events: vec![
10760                history_event(
10761                    "TimerScheduled",
10762                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10763                ),
10764                history_event(
10765                    "TimerFired",
10766                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10767                ),
10768            ],
10769            total_history_events: Some(2),
10770            history_size_bytes: None,
10771            continue_as_new_recommended: None,
10772            history_budget_pressure: None,
10773            next_history_page_token: None,
10774            workflow_task_attempt: 1,
10775            workflow_signal_id: None,
10776            signal_name: None,
10777            signal_arguments: None,
10778            workflow_update_id: None,
10779            update_name: None,
10780            lease_owner: Some("rust-worker".to_string()),
10781        };
10782
10783        let Error::NonDeterministicReplay(failure) = worker
10784            .execute_workflow_task(task)
10785            .expect_err("removed timer must fail replay")
10786        else {
10787            panic!("expected typed replay failure");
10788        };
10789        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10790        assert_eq!(failure.sequence, Some(1));
10791    }
10792
10793    #[test]
10794    fn workflow_context_emits_explicit_child_workflow_contract() {
10795        let ctx = WorkflowContext {
10796            state: Arc::new(Mutex::new(
10797                WorkflowState::new_with_identity(
10798                    Vec::new(),
10799                    Some("wf-parent".to_string()),
10800                    Some("run-parent".to_string()),
10801                    "parent-workers".to_string(),
10802                    JSON_CODEC.to_string(),
10803                    None,
10804                )
10805                .expect("workflow state"),
10806            )),
10807        };
10808        let options = ChildWorkflowOptions::new("python-workers")
10809            .parent_close_policy(ParentClosePolicy::RequestCancel)
10810            .retry_policy(ChildWorkflowRetryPolicy {
10811                max_attempts: Some(3),
10812                backoff_seconds: vec![1, 5],
10813                non_retryable_error_types: vec!["ValidationError".to_string()],
10814            })
10815            .execution_timeout_seconds(600)
10816            .run_timeout_seconds(120);
10817        let mut call = Box::pin(ctx.start_child_workflow(
10818            "python.fulfil-order",
10819            options,
10820            json!([{"order_id": "order-42"}]),
10821        ));
10822        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10823
10824        assert!(matches!(
10825            call.as_mut().poll(&mut task_context),
10826            Poll::Pending
10827        ));
10828        let commands = ctx.take_commands().expect("commands");
10829        assert_eq!(commands.len(), 1);
10830        let command = &commands[0];
10831        assert_eq!(command["type"], "start_child_workflow");
10832        assert_eq!(command["workflow_type"], "python.fulfil-order");
10833        assert_eq!(command["queue"], "python-workers");
10834        assert_eq!(command["parent_close_policy"], "request_cancel");
10835        assert_eq!(command["retry_policy"]["max_attempts"], 3);
10836        assert_eq!(command["execution_timeout_seconds"], 600);
10837        assert_eq!(command["run_timeout_seconds"], 120);
10838        assert_eq!(
10839            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
10840            json!([{"order_id": "order-42"}])
10841        );
10842    }
10843
10844    fn child_parent_worker() -> Worker {
10845        let client = Client::new("http://127.0.0.1:8080").expect("client");
10846        let mut worker = Worker::new(client, "rust-parent-workers");
10847        worker.register_workflow("rust.parent", |ctx, _input| async move {
10848            let child = ctx
10849                .start_child_workflow(
10850                    "python.child",
10851                    ChildWorkflowOptions::new("python-child-workers")
10852                        .parent_close_policy(ParentClosePolicy::Terminate),
10853                    json!([{"codec_probe": [1, true, "rust"]}]),
10854                )
10855                .await?;
10856            Ok(json!({
10857                "parent_workflow_id": child.parent.workflow_id,
10858                "parent_run_id": child.parent.run_id,
10859                "child_workflow_id": child.child.workflow_id,
10860                "child_run_id": child.child.run_id,
10861                "child_workflow_type": child.child_workflow_type,
10862                "result": child.result,
10863            }))
10864        });
10865        worker
10866    }
10867
10868    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
10869        WorkflowTask {
10870            task_id: "wft-child-parent".to_string(),
10871            workflow_id: Some("wf-parent".to_string()),
10872            run_id: Some("run-parent".to_string()),
10873            workflow_type: "rust.parent".to_string(),
10874            payload_codec: JSON_CODEC.to_string(),
10875            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10876            history_events: vec![
10877                HistoryEvent {
10878                    event_type: "ChildWorkflowScheduled".to_string(),
10879                    payload: json!({
10880                        "sequence": 1,
10881                        "child_call_id": "call-child",
10882                        "child_workflow_instance_id": "wf-child",
10883                        "child_workflow_run_id": "run-child",
10884                        "child_workflow_type": "python.child",
10885                    }),
10886                    raw: HashMap::new(),
10887                },
10888                HistoryEvent {
10889                    event_type: event_type.to_string(),
10890                    payload,
10891                    raw: HashMap::new(),
10892                },
10893            ],
10894            total_history_events: Some(2),
10895            history_size_bytes: None,
10896            continue_as_new_recommended: None,
10897            history_budget_pressure: None,
10898            next_history_page_token: None,
10899            workflow_task_attempt: 1,
10900            workflow_signal_id: None,
10901            signal_name: None,
10902            signal_arguments: None,
10903            workflow_update_id: None,
10904            update_name: None,
10905            lease_owner: Some("rust-worker".to_string()),
10906        }
10907    }
10908
10909    #[test]
10910    fn committed_child_result_replays_without_starting_a_duplicate() {
10911        let worker = child_parent_worker();
10912        let task = child_parent_task(
10913            "ChildRunCompleted",
10914            json!({
10915                "sequence": 1,
10916                "child_call_id": "call-child",
10917                "child_workflow_instance_id": "wf-child",
10918                "child_workflow_run_id": "run-child",
10919                "child_workflow_type": "python.child",
10920                "payload_codec": "json",
10921                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
10922            }),
10923        );
10924
10925        for _restart in 0..2 {
10926            let commands = worker
10927                .execute_workflow_task(task.clone())
10928                .expect("replayed parent task");
10929            assert_eq!(commands.len(), 1);
10930            assert_eq!(commands[0]["type"], "complete_workflow");
10931            assert!(!commands
10932                .iter()
10933                .any(|command| command["type"] == "start_child_workflow"));
10934            let output =
10935                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
10936            assert_eq!(output["parent_workflow_id"], "wf-parent");
10937            assert_eq!(output["parent_run_id"], "run-parent");
10938            assert_eq!(output["child_workflow_id"], "wf-child");
10939            assert_eq!(output["child_run_id"], "run-child");
10940            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
10941        }
10942    }
10943
10944    #[test]
10945    fn typed_child_arguments_and_results_survive_replay() {
10946        let client = Client::new("http://127.0.0.1:8080").expect("client");
10947        let mut worker = Worker::new(client, "rust-parent-workers");
10948        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
10949            let child = ctx
10950                .start_child_workflow_avro_value(
10951                    "python.typed-child",
10952                    ChildWorkflowOptions::new("python-workers"),
10953                    AvroValue::Array(vec![typed_fidelity_probe()]),
10954                )
10955                .await?;
10956            Ok(child.result)
10957        });
10958
10959        let initial = worker
10960            .execute_workflow_task(workflow_task(
10961                "rust.typed-parent",
10962                Vec::new(),
10963                DEFAULT_CODEC,
10964            ))
10965            .expect("typed child start");
10966        assert_eq!(initial[0]["type"], "start_child_workflow");
10967        assert_eq!(
10968            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
10969                .expect("typed child arguments"),
10970            AvroValue::Array(vec![typed_fidelity_probe()])
10971        );
10972
10973        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
10974            .expect("typed child result");
10975        let task = workflow_task(
10976            "rust.typed-parent",
10977            vec![
10978                history_event(
10979                    "ChildWorkflowScheduled",
10980                    json!({
10981                        "sequence": 1,
10982                        "child_call_id": "call-typed",
10983                        "child_workflow_instance_id": "wf-child",
10984                        "child_workflow_run_id": "run-child",
10985                        "child_workflow_type": "python.typed-child",
10986                    }),
10987                ),
10988                history_event(
10989                    "ChildRunCompleted",
10990                    json!({
10991                        "sequence": 1,
10992                        "child_call_id": "call-typed",
10993                        "child_workflow_instance_id": "wf-child",
10994                        "child_workflow_run_id": "run-child",
10995                        "child_workflow_type": "python.typed-child",
10996                        "payload_codec": DEFAULT_CODEC,
10997                        "result": result,
10998                    }),
10999                ),
11000            ],
11001            DEFAULT_CODEC,
11002        );
11003
11004        let commands = worker
11005            .execute_workflow_task(task)
11006            .expect("typed child replay");
11007        assert_eq!(commands[0]["type"], "complete_workflow");
11008        assert_eq!(
11009            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11010                .expect("typed parent result"),
11011            typed_fidelity_probe()
11012        );
11013    }
11014
11015    #[test]
11016    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11017        let worker = child_parent_worker();
11018        let mut task = child_parent_task("unused", Value::Null);
11019        task.history_events.truncate(1);
11020        task.total_history_events = Some(1);
11021
11022        for _redelivery_or_restart in 0..2 {
11023            let commands = worker
11024                .execute_workflow_task(task.clone())
11025                .expect("recorded child remains pending");
11026            assert!(
11027                commands.is_empty(),
11028                "recorded pending child must not be started again"
11029            );
11030        }
11031    }
11032
11033    #[test]
11034    fn child_cancellation_becomes_stable_parent_failure_command() {
11035        let worker = child_parent_worker();
11036        let task = child_parent_task(
11037            "ChildRunCancelled",
11038            json!({
11039                "sequence": 1,
11040                "child_workflow_instance_id": "wf-child",
11041                "child_workflow_run_id": "run-child",
11042                "child_workflow_type": "python.child",
11043                "failure_id": "failure-child",
11044                "failure_category": "cancelled",
11045                "message": "cancelled by parent-close policy",
11046            }),
11047        );
11048
11049        let commands = worker
11050            .execute_workflow_task(task)
11051            .expect("parent settlement");
11052        assert_eq!(commands.len(), 1);
11053        assert_eq!(commands[0]["type"], "fail_workflow");
11054        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11055        assert_eq!(
11056            commands[0]["exception"]["properties"]["reason"],
11057            "cancelled"
11058        );
11059        assert_eq!(
11060            commands[0]["exception"]["properties"]["child_workflow_run_id"],
11061            "run-child"
11062        );
11063    }
11064
11065    #[test]
11066    fn workflow_can_handle_typed_child_failure() {
11067        let client = Client::new("http://127.0.0.1:8080").expect("client");
11068        let mut worker = Worker::new(client, "rust-parent-workers");
11069        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11070            match ctx
11071                .start_child_workflow(
11072                    "python.child",
11073                    ChildWorkflowOptions::new("python-child-workers"),
11074                    json!([]),
11075                )
11076                .await
11077            {
11078                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11079                    "reason": failure.reason,
11080                    "failure_id": failure.failure_id,
11081                    "exception_class": failure.exception_class,
11082                    "child_run_id": failure.child_workflow_run_id,
11083                })),
11084                Err(error) => Err(error),
11085                Ok(_) => Err(Error::WorkerLoop(
11086                    "child unexpectedly succeeded".to_string(),
11087                )),
11088            }
11089        });
11090        let mut task = child_parent_task(
11091            "ChildRunFailed",
11092            json!({
11093                "sequence": 1,
11094                "child_workflow_instance_id": "wf-child",
11095                "child_workflow_run_id": "run-child",
11096                "child_workflow_type": "python.child",
11097                "failure_id": "failure-child",
11098                "failure_category": "child_workflow",
11099                "message": "payment rejected",
11100                "exception": {
11101                    "type": "PaymentRejected",
11102                    "class": "payments.PaymentRejected",
11103                    "message": "payment rejected"
11104                }
11105            }),
11106        );
11107        task.workflow_type = "rust.handled-parent".to_string();
11108
11109        let commands = worker.execute_workflow_task(task).expect("handled failure");
11110        assert_eq!(commands[0]["type"], "complete_workflow");
11111        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11112        assert_eq!(output["reason"], "child_workflow");
11113        assert_eq!(output["failure_id"], "failure-child");
11114        assert_eq!(output["exception_class"], "payments.PaymentRejected");
11115        assert_eq!(output["child_run_id"], "run-child");
11116    }
11117
11118    #[test]
11119    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11120        let client = Client::new("http://127.0.0.1:8080").expect("client");
11121        let mut worker = Worker::new(client, "rust-workers");
11122
11123        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11124            let signal = ctx.wait_signal("start").await?;
11125            let name = signal
11126                .first()
11127                .and_then(|value| value.as_str())
11128                .unwrap_or("world");
11129            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11130            Ok(json!({
11131                "greeting": greeting,
11132                "language": "rust"
11133            }))
11134        });
11135
11136        let signal_arguments =
11137            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11138        let task = WorkflowTask {
11139            task_id: "wft-rust-signal-1".to_string(),
11140            workflow_id: Some("wf-rust-hello".to_string()),
11141            run_id: Some("run-rust-hello".to_string()),
11142            workflow_type: "rust.hello_workflow".to_string(),
11143            payload_codec: DEFAULT_CODEC.to_string(),
11144            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11145            history_events: vec![HistoryEvent {
11146                event_type: "SignalReceived".to_string(),
11147                payload: json!({
11148                    "signal_id": "sig-rust-1",
11149                    "signal_name": "start"
11150                }),
11151                raw: HashMap::new(),
11152            }],
11153            total_history_events: Some(1),
11154            history_size_bytes: None,
11155            continue_as_new_recommended: None,
11156            history_budget_pressure: None,
11157            next_history_page_token: None,
11158            workflow_task_attempt: 1,
11159            workflow_signal_id: Some("sig-rust-1".to_string()),
11160            signal_name: Some("start".to_string()),
11161            signal_arguments: Some(signal_arguments),
11162            workflow_update_id: None,
11163            update_name: None,
11164            lease_owner: Some("rust-worker".to_string()),
11165        };
11166
11167        let commands = worker.execute_workflow_task(task).expect("workflow task");
11168
11169        assert_eq!(commands.len(), 1);
11170        assert_eq!(commands[0]["type"], "schedule_activity");
11171        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11172        assert_eq!(
11173            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11174            json!(["Rust"])
11175        );
11176    }
11177
11178    #[test]
11179    fn workflow_task_appends_paginated_history_events() {
11180        let mut task = WorkflowTask {
11181            task_id: "wft-rust-pages-1".to_string(),
11182            workflow_id: Some("wf-rust-pages".to_string()),
11183            run_id: Some("run-rust-pages".to_string()),
11184            workflow_type: "rust.hello_workflow".to_string(),
11185            payload_codec: DEFAULT_CODEC.to_string(),
11186            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11187            history_events: vec![HistoryEvent {
11188                event_type: "WorkflowStarted".to_string(),
11189                payload: json!({}),
11190                raw: HashMap::new(),
11191            }],
11192            total_history_events: Some(3),
11193            history_size_bytes: None,
11194            continue_as_new_recommended: None,
11195            history_budget_pressure: None,
11196            next_history_page_token: Some("MQ==".to_string()),
11197            workflow_task_attempt: 1,
11198            workflow_signal_id: None,
11199            signal_name: None,
11200            signal_arguments: None,
11201            workflow_update_id: None,
11202            update_name: None,
11203            lease_owner: Some("rust-worker".to_string()),
11204        };
11205
11206        task.append_history_page(WorkflowTaskHistoryPage {
11207            history_events: vec![
11208                HistoryEvent {
11209                    event_type: "SignalReceived".to_string(),
11210                    payload: json!({
11211                        "signal_id": "sig-rust-1",
11212                        "signal_name": "start",
11213                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11214                            .expect("signal arguments")
11215                    }),
11216                    raw: HashMap::new(),
11217                },
11218                HistoryEvent {
11219                    event_type: "MarkerRecorded".to_string(),
11220                    payload: json!({"sequence": 3}),
11221                    raw: HashMap::new(),
11222                },
11223            ],
11224            total_history_events: Some(3),
11225            next_history_page_token: None,
11226        });
11227
11228        assert_eq!(task.history_events.len(), 3);
11229        assert_eq!(task.total_history_events, Some(3));
11230        assert_eq!(task.next_history_page_token, None);
11231
11232        let signal = task
11233            .history_events
11234            .iter()
11235            .find(|event| event.event_type == "SignalReceived")
11236            .expect("signal event");
11237        assert_eq!(
11238            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11239            vec![AvroValue::String("Rust".to_string())]
11240        );
11241    }
11242
11243    #[tokio::test]
11244    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11245        let client = Client::new("http://127.0.0.1:8080").expect("client");
11246        let mut worker = Worker::new(client, "rust-workers");
11247        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11248        worker.register_query("counter", "current", |ctx, _args| async move {
11249            let mut count = 0_i64;
11250            for signal in ctx.signal_events() {
11251                let value = signal
11252                    .arguments
11253                    .first()
11254                    .and_then(Value::as_i64)
11255                    .unwrap_or_default();
11256                match signal.name.as_str() {
11257                    "increment" => count += value,
11258                    "set" => count = value,
11259                    _ => {}
11260                }
11261            }
11262            Ok(json!(count))
11263        });
11264
11265        let task = QueryTask {
11266            query_task_id: "query-rust-counter".to_string(),
11267            query_task_attempt: 1,
11268            lease_owner: Some("rust-worker".to_string()),
11269            workflow_id: Some("counter-1".to_string()),
11270            run_id: Some("run-counter-1".to_string()),
11271            workflow_type: "counter".to_string(),
11272            query_name: "current".to_string(),
11273            payload_codec: DEFAULT_CODEC.to_string(),
11274            workflow_arguments: Some(
11275                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11276            ),
11277            query_arguments: Some(
11278                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11279            ),
11280            history_events: vec![
11281                HistoryEvent {
11282                    event_type: "SignalReceived".to_string(),
11283                    payload: json!({
11284                        "signal_id": "php-signal-1",
11285                        "signal_name": "increment",
11286                        "workflow_sequence": 1,
11287                        "payload_codec": DEFAULT_CODEC,
11288                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11289                    }),
11290                    raw: HashMap::new(),
11291                },
11292                HistoryEvent {
11293                    event_type: "SignalReceived".to_string(),
11294                    payload: json!({
11295                        "signal_id": "python-signal-2",
11296                        "signal_name": "increment",
11297                        "workflow_sequence": 2,
11298                        "payload_codec": JSON_CODEC,
11299                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11300                    }),
11301                    raw: HashMap::new(),
11302                },
11303                HistoryEvent {
11304                    event_type: "SignalReceived".to_string(),
11305                    payload: json!({
11306                        "signal_id": "rust-signal-3",
11307                        "signal_name": "set",
11308                        "workflow_sequence": 3,
11309                        "payload_codec": DEFAULT_CODEC,
11310                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11311                    }),
11312                    raw: HashMap::new(),
11313                },
11314            ],
11315            history_export: None,
11316            run_status: Some("completed".to_string()),
11317        };
11318
11319        let result = worker.execute_query_task(task).await.expect("query result");
11320        assert_eq!(result.into_json().expect("query projection"), json!(0));
11321    }
11322
11323    #[tokio::test]
11324    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11325        let worker = replay_counter_worker();
11326        let running_history = json!([
11327            {
11328                "type": "ActivityCompleted",
11329                "payload": {
11330                    "sequence": 1,
11331                    "activity_type": "load-counter",
11332                    "payload_codec": "json",
11333                    "result": {"codec": "json", "blob": "\"loaded\""}
11334                }
11335            },
11336            {
11337                "type": "SignalWaitOpened",
11338                "payload": {
11339                    "sequence": 3,
11340                    "signal_name": "increment"
11341                }
11342            },
11343            {
11344                "type": "SignalReceived",
11345                "payload": {
11346                    "signal_id": "signal-3",
11347                    "signal_name": "increment",
11348                    "workflow_sequence": 2,
11349                    "payload_codec": "json",
11350                    "arguments": {"codec": "json", "blob": "[3]"}
11351                }
11352            },
11353            {
11354                "type": "SignalApplied",
11355                "payload": {
11356                    "sequence": 3,
11357                    "signal_id": "signal-3",
11358                    "signal_name": "increment",
11359                    "payload_codec": "json",
11360                    "value": {"codec": "json", "blob": "[3]"}
11361                }
11362            }
11363        ]);
11364
11365        let running = worker
11366            .execute_query_task(replay_counter_query(
11367                "current",
11368                running_history.clone(),
11369                "running",
11370            ))
11371            .await
11372            .expect("running replay query");
11373        assert_eq!(
11374            running.clone().into_json().expect("query projection"),
11375            json!({"loaded": "loaded", "count": 3, "finished": false})
11376        );
11377
11378        let detached = worker
11379            .execute_query_task(replay_counter_query(
11380                "detached-mutation",
11381                running_history.clone(),
11382                "running",
11383            ))
11384            .await
11385            .expect("query mutates only its detached state clone");
11386        assert_eq!(detached.into_json().expect("query projection"), json!(999));
11387        let failed = worker
11388            .execute_query_task(replay_counter_query(
11389                "failed-mutation",
11390                running_history.clone(),
11391                "running",
11392            ))
11393            .await
11394            .expect_err("failed query");
11395        assert_eq!(failed.reason, "query_rejected");
11396        let unchanged = worker
11397            .execute_query_task(replay_counter_query("current", running_history, "running"))
11398            .await
11399            .expect("later query reconstructs unchanged state");
11400        assert_eq!(unchanged, running);
11401
11402        let restarted_worker = replay_counter_worker();
11403        let restarted_task: QueryTask = serde_json::from_value(json!({
11404            "query_task_id": "query-after-restart",
11405            "workflow_id": "counter-1",
11406            "run_id": "run-counter-1",
11407            "workflow_type": "replay-counter",
11408            "query_name": "current",
11409            "payload_codec": "json",
11410            "workflow_arguments": {"codec": "json", "blob": "[]"},
11411            "query_arguments": {"codec": "json", "blob": "[]"},
11412            "history_events": [],
11413            "history_export": {
11414                "payloads": {"codec": "json"},
11415                "history_events": [
11416                    {
11417                        "type": "ActivityCompleted",
11418                        "payload": {
11419                            "sequence": 1,
11420                            "activity_type": "load-counter",
11421                            "payload_codec": "json",
11422                            "result": null
11423                        }
11424                    },
11425                    {
11426                        "type": "SignalWaitOpened",
11427                        "payload": {
11428                            "sequence": 3,
11429                            "signal_name": "increment"
11430                        }
11431                    },
11432                    {
11433                        "type": "SignalReceived",
11434                        "payload": {
11435                            "signal_id": "signal-3",
11436                            "signal_name": "increment",
11437                            "workflow_sequence": 2
11438                        }
11439                    },
11440                    {
11441                        "type": "SignalApplied",
11442                        "payload": {
11443                            "sequence": 3,
11444                            "signal_id": "signal-3",
11445                            "signal_name": "increment"
11446                        }
11447                    },
11448                    {
11449                        "type": "SignalWaitOpened",
11450                        "payload": {
11451                            "sequence": 5,
11452                            "signal_name": "increment"
11453                        }
11454                    },
11455                    {
11456                        "type": "SignalReceived",
11457                        "payload": {
11458                            "signal_id": "signal-5",
11459                            "signal_name": "increment",
11460                            "workflow_sequence": 4
11461                        }
11462                    },
11463                    {
11464                        "type": "SignalApplied",
11465                        "payload": {
11466                            "sequence": 5,
11467                            "signal_id": "signal-5",
11468                            "signal_name": "increment"
11469                        }
11470                    }
11471                ],
11472                "activities": [{
11473                    "sequence": 1,
11474                    "activity_type": "load-counter",
11475                    "payload_codec": "json",
11476                    "result": {"codec": "json", "blob": "\"loaded\""}
11477                }],
11478                "signals": [
11479                    {
11480                        "id": "signal-3",
11481                        "name": "increment",
11482                        "workflow_sequence": 2,
11483                        "payload_codec": "json",
11484                        "arguments": "[3]"
11485                    },
11486                    {
11487                        "id": "signal-5",
11488                        "name": "increment",
11489                        "workflow_sequence": 4,
11490                        "payload_codec": "json",
11491                        "arguments": "[5]"
11492                    }
11493                ]
11494            },
11495            "run_status": "completed"
11496        }))
11497        .expect("cold replay query task");
11498        let completed = restarted_worker
11499            .execute_query_task(restarted_task)
11500            .await
11501            .expect("completed cold replay query");
11502        assert_eq!(
11503            completed.into_json().expect("query projection"),
11504            json!({"loaded": "loaded", "count": 8, "finished": true})
11505        );
11506    }
11507
11508    #[tokio::test]
11509    async fn replayed_query_replay_failures_are_machine_readable() {
11510        let worker = replay_counter_worker();
11511        let task = replay_counter_query(
11512            "current",
11513            json!([{
11514                "type": "ActivityCompleted",
11515                "payload": {
11516                    "sequence": 1,
11517                    "payload_codec": "json",
11518                    "result": {"codec": "json", "blob": "{"}
11519                }
11520            }]),
11521            "running",
11522        );
11523        let failure = worker
11524            .execute_query_task(task)
11525            .await
11526            .expect_err("invalid replay history payload");
11527        assert_eq!(failure.reason, "query_workflow_state_unavailable");
11528        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11529    }
11530
11531    #[tokio::test]
11532    async fn query_task_restores_compact_history_from_export() {
11533        let client = Client::new("http://127.0.0.1:8080").expect("client");
11534        let mut worker = Worker::new(client, "rust-workers");
11535        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11536        worker.register_query("counter", "current", |ctx, _args| async move {
11537            Ok(json!(ctx.signals("increment")[0][0]))
11538        });
11539        let task: QueryTask = serde_json::from_value(json!({
11540            "query_task_id": "query-export",
11541            "workflow_type": "counter",
11542            "query_name": "current",
11543            "payload_codec": "json",
11544            "workflow_arguments": {"codec": "json", "blob": "[]"},
11545            "query_arguments": {"codec": "json", "blob": "[]"},
11546            "history_events": [],
11547            "history_export": {
11548                "payloads": {"codec": "json"},
11549                "history_events": [{
11550                    "type": "SignalReceived",
11551                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11552                }],
11553                "signals": [{
11554                    "id": "signal-export",
11555                    "name": "increment",
11556                    "status": "applied",
11557                    "workflow_sequence": 1,
11558                    "payload_codec": "json",
11559                    "arguments": "[9]"
11560                }]
11561            }
11562        }))
11563        .expect("query task");
11564
11565        let result = worker.execute_query_task(task).await.expect("query result");
11566        assert_eq!(result.into_json().expect("query projection"), json!(9));
11567    }
11568
11569    #[tokio::test]
11570    async fn query_task_failures_have_stable_reasons() {
11571        let client = Client::new("http://127.0.0.1:8080").expect("client");
11572        let mut worker = Worker::new(client, "rust-workers");
11573        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11574        worker.register_query(
11575            "counter",
11576            "current",
11577            |_ctx, _args| async move { Ok(json!(0)) },
11578        );
11579
11580        let base_task = QueryTask {
11581            query_task_id: "query-errors".to_string(),
11582            query_task_attempt: 1,
11583            lease_owner: None,
11584            workflow_id: Some("counter-errors".to_string()),
11585            run_id: Some("run-errors".to_string()),
11586            workflow_type: "counter".to_string(),
11587            query_name: "missing".to_string(),
11588            payload_codec: JSON_CODEC.to_string(),
11589            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11590            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11591            history_events: Vec::new(),
11592            history_export: None,
11593            run_status: Some("running".to_string()),
11594        };
11595
11596        let unknown = worker
11597            .execute_query_task(base_task.clone())
11598            .await
11599            .expect_err("unknown query");
11600        assert_eq!(unknown.reason, "rejected_unknown_query");
11601
11602        let mut malformed = base_task;
11603        malformed.query_name = "current".to_string();
11604        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11605        let malformed = worker
11606            .execute_query_task(malformed)
11607            .await
11608            .expect_err("malformed payload");
11609        assert_eq!(malformed.reason, "query_payload_decode_failed");
11610
11611        let client = Client::new("http://127.0.0.1:8080").expect("client");
11612        let mut unavailable_worker = Worker::new(client, "rust-workers");
11613        unavailable_worker
11614            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11615        let unavailable_task: QueryTask = serde_json::from_value(json!({
11616            "query_task_id": "query-unavailable",
11617            "workflow_type": "counter",
11618            "query_name": "current",
11619            "payload_codec": "json",
11620            "workflow_arguments": {"codec": "json", "blob": "[]"},
11621            "query_arguments": {"codec": "json", "blob": "[]"}
11622        }))
11623        .expect("query task");
11624        let unavailable = unavailable_worker
11625            .execute_query_task(unavailable_task)
11626            .await
11627            .expect_err("query handler unavailable");
11628        assert_eq!(unavailable.reason, "query_handler_unavailable");
11629    }
11630
11631    #[tokio::test]
11632    async fn client_query_decodes_result_and_typed_failure() {
11633        let server = MockWorkerServer::start();
11634        let client = Client::builder(server.base_url())
11635            .timeout(Duration::from_secs(2))
11636            .build()
11637            .expect("client");
11638
11639        let result = client
11640            .query_workflow("counter-1", "current", json!([]))
11641            .await
11642            .expect("query result");
11643        assert_eq!(result, json!({"count": 8}));
11644
11645        let error = client
11646            .query_workflow("counter-1", "missing", json!([]))
11647            .await
11648            .expect_err("unknown query");
11649        let Error::QueryFailed(failure) = error else {
11650            panic!("expected typed query failure");
11651        };
11652        assert_eq!(failure.status, 404);
11653        assert_eq!(failure.reason, "rejected_unknown_query");
11654    }
11655
11656    #[tokio::test]
11657    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11658        let server = MockWorkerServer::start();
11659        let client = Client::builder(server.base_url())
11660            .timeout(Duration::from_secs(2))
11661            .build()
11662            .expect("client");
11663        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11664
11665        client
11666            .start_workflow(
11667                "typed.echo",
11668                "rust-workers",
11669                "typed-start",
11670                arguments.clone(),
11671            )
11672            .await
11673            .expect("typed workflow start");
11674        assert_eq!(
11675            decode_wire_avro_value(
11676                &server.request_body("/api/workflows")["input"],
11677                DEFAULT_CODEC,
11678            )
11679            .expect("typed start input"),
11680            arguments
11681        );
11682
11683        client
11684            .signal_workflow("typed-1", "changed", arguments.clone())
11685            .await
11686            .expect("typed signal");
11687        assert_eq!(
11688            decode_wire_avro_value(
11689                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11690                DEFAULT_CODEC,
11691            )
11692            .expect("typed signal input"),
11693            arguments
11694        );
11695
11696        assert_eq!(
11697            client
11698                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11699                .await
11700                .expect("typed query"),
11701            typed_fidelity_probe()
11702        );
11703        assert_eq!(
11704            decode_wire_avro_value(
11705                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11706                DEFAULT_CODEC,
11707            )
11708            .expect("typed query input"),
11709            arguments
11710        );
11711
11712        assert_eq!(
11713            client
11714                .update_workflow_avro_value(
11715                    "typed-1",
11716                    "replace",
11717                    arguments.clone(),
11718                    Some("typed-request"),
11719                )
11720                .await
11721                .expect("typed update"),
11722            typed_fidelity_probe()
11723        );
11724        let update = server.request_body("/api/workflows/typed-1/update/replace");
11725        assert_eq!(update["request_id"], "typed-request");
11726        assert_eq!(
11727            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11728            arguments
11729        );
11730
11731        let handle = WorkflowHandle {
11732            client: client.clone(),
11733            workflow_id: "typed-1".to_string(),
11734            run_id: Some("run-typed-1".to_string()),
11735            workflow_type: "typed.echo".to_string(),
11736        };
11737        assert_eq!(
11738            handle
11739                .result_avro_value(WorkflowResultOptions::default())
11740                .await
11741                .expect("typed workflow result"),
11742            typed_fidelity_probe()
11743        );
11744
11745        client
11746            .complete_activity_task(
11747                "activity-typed",
11748                "attempt-typed",
11749                "rust-worker",
11750                typed_fidelity_probe(),
11751                DEFAULT_CODEC,
11752            )
11753            .await
11754            .expect("typed activity completion");
11755        assert_eq!(
11756            decode_wire_avro_value(
11757                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11758                    ["result"],
11759                DEFAULT_CODEC,
11760            )
11761            .expect("typed activity result"),
11762            typed_fidelity_probe()
11763        );
11764        client
11765            .fail_activity_task(
11766                "activity-typed",
11767                "attempt-typed",
11768                "rust-worker",
11769                "typed failure",
11770                true,
11771            )
11772            .await
11773            .expect("activity failure");
11774    }
11775
11776    #[tokio::test]
11777    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11778        let server = MockWorkerServer::start();
11779        let client = Client::builder(server.base_url())
11780            .timeout(Duration::from_secs(2))
11781            .build()
11782            .expect("client");
11783
11784        let options = WorkflowCommandOptions::new()
11785            .reason("cleanup requested")
11786            .request_id("cancel-17");
11787        let cancelled = client
11788            .cancel_workflow("wf-lifecycle", options)
11789            .await
11790            .expect("instance cancellation");
11791        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11792        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11793        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11794        assert_eq!(
11795            server.request_body("/api/workflows/wf-lifecycle/cancel"),
11796            json!({"reason":"cleanup requested","request_id":"cancel-17"})
11797        );
11798
11799        let terminated = client
11800            .terminate_workflow(
11801                "wf-lifecycle",
11802                WorkflowCommandOptions::new().reason("forced stop"),
11803            )
11804            .await
11805            .expect("instance termination");
11806        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11807        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11808
11809        client
11810            .cancel_workflow_run(
11811                "wf-lifecycle",
11812                "run-current",
11813                WorkflowCommandOptions::default(),
11814            )
11815            .await
11816            .expect("selected run cancellation");
11817        client
11818            .terminate_workflow_run(
11819                "wf-lifecycle",
11820                "run-current",
11821                WorkflowCommandOptions::default(),
11822            )
11823            .await
11824            .expect("selected run termination");
11825
11826        for (command, error) in [
11827            (
11828                WorkflowCommandKind::Cancel,
11829                client
11830                    .cancel_workflow_run(
11831                        "wf-lifecycle",
11832                        "run-stale",
11833                        WorkflowCommandOptions::default(),
11834                    )
11835                    .await
11836                    .expect_err("stale cancellation must be rejected"),
11837            ),
11838            (
11839                WorkflowCommandKind::Terminate,
11840                client
11841                    .terminate_workflow_run(
11842                        "wf-lifecycle",
11843                        "run-stale",
11844                        WorkflowCommandOptions::default(),
11845                    )
11846                    .await
11847                    .expect_err("stale termination must be rejected"),
11848            ),
11849        ] {
11850            let Error::WorkflowCommandRejected(rejection) = error else {
11851                panic!("expected typed command rejection");
11852            };
11853            assert_eq!(rejection.command, command);
11854            assert_eq!(rejection.status, 409);
11855            assert_eq!(rejection.reason, "historical_run_command_rejected");
11856            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
11857            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
11858        }
11859    }
11860
11861    #[tokio::test]
11862    async fn workflow_start_options_send_server_enforced_deadlines() {
11863        let server = MockWorkerServer::start();
11864        let client = Client::builder(server.base_url())
11865            .timeout(Duration::from_secs(2))
11866            .build()
11867            .expect("client");
11868
11869        let handle = client
11870            .start_workflow_with_options(
11871                "rust.timeout",
11872                "rust-timeouts",
11873                "wf-start-options",
11874                WorkflowStartOptions::new()
11875                    .execution_timeout_seconds(30)
11876                    .run_timeout_seconds(1),
11877                json!([]),
11878            )
11879            .await
11880            .expect("workflow start");
11881
11882        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
11883        let body = server.request_body("/api/workflows");
11884        assert_eq!(body["execution_timeout_seconds"], 30);
11885        assert_eq!(body["run_timeout_seconds"], 1);
11886
11887        let invalid = client
11888            .start_workflow_with_options(
11889                "rust.timeout",
11890                "rust-timeouts",
11891                "wf-invalid-options",
11892                WorkflowStartOptions::new()
11893                    .execution_timeout_seconds(1)
11894                    .run_timeout_seconds(2),
11895                json!([]),
11896            )
11897            .await
11898            .expect_err("invalid deadline ordering");
11899        assert!(invalid
11900            .to_string()
11901            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
11902    }
11903
11904    #[tokio::test]
11905    async fn workflow_result_returns_each_typed_terminal_outcome() {
11906        let server = MockWorkerServer::start();
11907        let client = Client::builder(server.base_url())
11908            .timeout(Duration::from_secs(2))
11909            .build()
11910            .expect("client");
11911        let options = WorkflowResultOptions {
11912            poll_interval: Duration::ZERO,
11913            timeout: Duration::from_secs(1),
11914        };
11915
11916        let failed = WorkflowHandle {
11917            client: client.clone(),
11918            workflow_id: "wf-failed".to_string(),
11919            run_id: Some("run-failed".to_string()),
11920            workflow_type: "failure".to_string(),
11921        }
11922        .result(options)
11923        .await
11924        .expect_err("failed outcome");
11925        let Error::WorkflowFailed(failure) = failed else {
11926            panic!("expected WorkflowFailed");
11927        };
11928        assert_eq!(failure.workflow_id, "wf-failed");
11929        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
11930        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
11931        assert_eq!(failure.failure_category.as_deref(), Some("application"));
11932        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
11933        assert_eq!(
11934            failure.exception_class.as_deref(),
11935            Some("billing::PaymentError")
11936        );
11937        assert_eq!(failure.non_retryable, Some(true));
11938
11939        for (workflow_id, expected_kind, expected_reason) in [
11940            (
11941                "wf-cancelled",
11942                WorkflowTerminalKind::Cancelled,
11943                "cleanup requested",
11944            ),
11945            (
11946                "wf-terminated",
11947                WorkflowTerminalKind::Terminated,
11948                "forced stop",
11949            ),
11950            (
11951                "wf-timed-out",
11952                WorkflowTerminalKind::TimedOut,
11953                "run_timeout",
11954            ),
11955        ] {
11956            let error = WorkflowHandle {
11957                client: client.clone(),
11958                workflow_id: workflow_id.to_string(),
11959                run_id: None,
11960                workflow_type: "terminal".to_string(),
11961            }
11962            .result(options)
11963            .await
11964            .expect_err("typed terminal outcome");
11965            let outcome = match error {
11966                Error::WorkflowCancelled(outcome) => outcome,
11967                Error::WorkflowTerminated(outcome) => outcome,
11968                Error::WorkflowTimedOut(outcome) => outcome,
11969                other => panic!("unexpected terminal error: {other}"),
11970            };
11971            assert_eq!(outcome.kind, expected_kind);
11972            assert_eq!(outcome.workflow_id, workflow_id);
11973            assert_eq!(outcome.reason, expected_reason);
11974        }
11975
11976        let wait_timeout = WorkflowHandle {
11977            client,
11978            workflow_id: "wf-waiting".to_string(),
11979            run_id: Some("run-waiting".to_string()),
11980            workflow_type: "waiting".to_string(),
11981        }
11982        .result(WorkflowResultOptions {
11983            poll_interval: Duration::ZERO,
11984            timeout: Duration::ZERO,
11985        })
11986        .await
11987        .expect_err("client wait timeout");
11988        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
11989            panic!("expected typed client timeout");
11990        };
11991        assert_eq!(timeout.reason, "result_wait_timeout");
11992        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
11993        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
11994    }
11995
11996    #[tokio::test]
11997    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
11998        let server = MockWorkerServer::start();
11999        let client = Client::builder(server.base_url())
12000            .timeout(Duration::from_secs(2))
12001            .build()
12002            .expect("client");
12003
12004        let handle = WorkflowHandle {
12005            client,
12006            workflow_id: "wf-selected".to_string(),
12007            run_id: Some("run-selected".to_string()),
12008            workflow_type: "selected".to_string(),
12009        };
12010        let options = WorkflowResultOptions {
12011            poll_interval: Duration::ZERO,
12012            timeout: Duration::from_secs(1),
12013        };
12014
12015        let current = handle
12016            .result(options)
12017            .await
12018            .expect("instance result follows the current run");
12019        assert_eq!(current, json!("current run output"));
12020
12021        let error = handle
12022            .result_selected_run(options)
12023            .await
12024            .expect_err("the selected run is cancelled even though the current run completed");
12025
12026        let Error::WorkflowCancelled(outcome) = error else {
12027            panic!("expected selected run cancellation");
12028        };
12029        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12030        assert_eq!(outcome.reason, "selected run cancelled");
12031        assert_eq!(
12032            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12033            1
12034        );
12035        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12036    }
12037
12038    #[tokio::test]
12039    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12040        let server = MockWorkerServer::draining_polls();
12041        let client = Client::builder(server.base_url())
12042            .timeout(Duration::from_secs(2))
12043            .build()
12044            .expect("client");
12045
12046        let workflow = client
12047            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12048            .await
12049            .expect("workflow drain response");
12050        let activity = client
12051            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12052            .await
12053            .expect("activity drain response");
12054        let query = client
12055            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12056            .await
12057            .expect("query drain response");
12058
12059        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12060            assert_eq!(
12061                outcome,
12062                WorkerPollOutcome::Stop {
12063                    poll_status: Some("draining".to_string()),
12064                    reason: Some("worker_draining".to_string()),
12065                }
12066            );
12067        }
12068
12069        assert!(client
12070            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12071            .await
12072            .expect("compatibility poll")
12073            .is_none());
12074    }
12075
12076    #[tokio::test]
12077    async fn managed_worker_honors_drain_stop_for_every_task_family() {
12078        let server = MockWorkerServer::draining_polls();
12079        let client = Client::builder(server.base_url())
12080            .timeout(Duration::from_secs(2))
12081            .build()
12082            .expect("client");
12083
12084        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12085            .worker_id("draining-workflow-worker")
12086            .poll_timeout(Duration::ZERO);
12087        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12088        workflow_worker
12089            .run()
12090            .await
12091            .expect("workflow drain is a clean stop");
12092
12093        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12094            .worker_id("draining-activity-worker")
12095            .poll_timeout(Duration::ZERO);
12096        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12097        activity_worker
12098            .run()
12099            .await
12100            .expect("activity drain is a clean stop");
12101
12102        let mut query_worker = Worker::new(client, "rust-workers")
12103            .worker_id("draining-query-worker")
12104            .poll_timeout(Duration::ZERO);
12105        query_worker.register_query("counter", "current", |_ctx, _args| async {
12106            Ok(Value::Null)
12107        });
12108        query_worker
12109            .run()
12110            .await
12111            .expect("query drain is a clean stop");
12112    }
12113
12114    #[tokio::test]
12115    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12116        let server = MockWorkerServer::start();
12117        let client = Client::builder(server.base_url())
12118            .timeout(Duration::from_secs(2))
12119            .build()
12120            .expect("client");
12121
12122        let heartbeat = client
12123            .heartbeat_activity_task(
12124                "activity-cancel",
12125                "attempt-cancel",
12126                "rust-worker",
12127                typed_fidelity_probe(),
12128            )
12129            .await
12130            .expect("cancellation heartbeat");
12131        assert!(heartbeat.cancel_requested);
12132        assert!(heartbeat.should_stop());
12133        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12134        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12135        let heartbeat_body =
12136            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12137        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12138        assert_eq!(
12139            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12140                .expect("typed heartbeat details"),
12141            typed_fidelity_probe()
12142        );
12143
12144        let error = client
12145            .complete_activity_task(
12146                "activity-cancel",
12147                "attempt-cancel",
12148                "rust-worker",
12149                json!({"late":true}),
12150                JSON_CODEC,
12151            )
12152            .await
12153            .expect_err("late completion must be refused");
12154        assert!(activity_task_rejection_is_final(&error));
12155        let Error::ActivityTaskRejected(rejection) = error else {
12156            panic!("expected typed activity rejection");
12157        };
12158        assert_eq!(rejection.status, 409);
12159        assert_eq!(rejection.reason, "run_cancelled");
12160        assert!(rejection.cancel_requested);
12161        assert_eq!(rejection.can_continue, Some(false));
12162    }
12163
12164    #[tokio::test]
12165    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12166        let server = MockWorkerServer::cancelled_activity();
12167        let client = Client::builder(server.base_url())
12168            .timeout(Duration::from_secs(2))
12169            .build()
12170            .expect("client");
12171        let cancellation_observed = Arc::new(AtomicBool::new(false));
12172        let observed = Arc::clone(&cancellation_observed);
12173        let mut worker = Worker::new(client.clone(), "rust-workers")
12174            .worker_id("rust-cancel-worker")
12175            .poll_timeout(Duration::from_millis(10));
12176        worker.register_activity("cancel-aware", move |ctx, _args| {
12177            let observed = Arc::clone(&observed);
12178            async move {
12179                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12180                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12181                Ok(json!({"late":"completion"}))
12182            }
12183        });
12184
12185        assert_eq!(
12186            worker.run_once().await.expect("cancelled attempt handled"),
12187            1
12188        );
12189        assert!(cancellation_observed.load(Ordering::SeqCst));
12190        assert_eq!(
12191            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12192            1
12193        );
12194
12195        let mut restarted = Worker::new(client, "rust-workers")
12196            .worker_id("rust-cancel-worker-restarted")
12197            .poll_timeout(Duration::from_millis(10));
12198        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12199        assert_eq!(
12200            restarted
12201                .run_once()
12202                .await
12203                .expect("replacement worker continues polling"),
12204            0
12205        );
12206    }
12207
12208    #[tokio::test]
12209    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12210        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12211        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12212        let client = Client::builder(server.base_url())
12213            .timeout(Duration::from_secs(2))
12214            .build()
12215            .expect("client");
12216
12217        let direct_error = client
12218            .complete_workflow_task(
12219                "workflow-timeout-task",
12220                "timeout-worker",
12221                3,
12222                vec![json!({"type": "complete_workflow", "result": null})],
12223            )
12224            .await
12225            .expect_err("the low-level client preserves the completion rejection");
12226        let Error::Http { status, body } = direct_error else {
12227            panic!("expected the original HTTP completion rejection");
12228        };
12229        assert_eq!(status, reqwest::StatusCode::CONFLICT);
12230        assert_eq!(
12231            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12232            "run_timed_out"
12233        );
12234
12235        let mut worker = Worker::new(client, "rust-workers")
12236            .worker_id("timeout-worker")
12237            .poll_timeout(Duration::from_millis(10));
12238        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12239            Ok(json!({"late": "result"}))
12240        });
12241
12242        assert_eq!(
12243            worker
12244                .run_once()
12245                .await
12246                .expect("authoritative selected-run timeout settles the tick"),
12247            1
12248        );
12249        assert_eq!(
12250            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12251            2,
12252            "both the direct client proof and managed worker must see the rejection"
12253        );
12254    }
12255
12256    #[tokio::test]
12257    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12258        for (name, status, response) in [
12259            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12260            (
12261                "command was recorded",
12262                "409 Conflict",
12263                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12264            ),
12265            (
12266                "lease conflict",
12267                "409 Conflict",
12268                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12269            ),
12270            (
12271                "nonterminal run",
12272                "409 Conflict",
12273                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12274            ),
12275            (
12276                "different selected run",
12277                "409 Conflict",
12278                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12279            ),
12280            (
12281                "different task attempt",
12282                "409 Conflict",
12283                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12284            ),
12285            (
12286                "authentication failure",
12287                "401 Unauthorized",
12288                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12289            ),
12290            (
12291                "authorization failure",
12292                "403 Forbidden",
12293                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12294            ),
12295            (
12296                "protocol failure",
12297                "400 Bad Request",
12298                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12299            ),
12300            (
12301                "malformed command",
12302                "422 Unprocessable Entity",
12303                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12304            ),
12305            (
12306                "transient server failure",
12307                "503 Service Unavailable",
12308                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12309            ),
12310        ] {
12311            let server = MockWorkerServer::workflow_completion(status, response);
12312            let client = Client::builder(server.base_url())
12313                .timeout(Duration::from_secs(2))
12314                .build()
12315                .expect("client");
12316            let mut worker = Worker::new(client, "rust-workers")
12317                .worker_id("timeout-worker")
12318                .poll_timeout(Duration::from_millis(10));
12319            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12320                Ok(json!({"late": "result"}))
12321            });
12322
12323            let error = worker
12324                .run_once()
12325                .await
12326                .expect_err(&format!("{name} must remain an error"));
12327            assert!(
12328                matches!(error, Error::Http { .. } | Error::Protocol(_)),
12329                "{name} returned an unexpected error variant: {error}"
12330            );
12331        }
12332    }
12333
12334    #[tokio::test]
12335    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12336        let server = MockWorkerServer::start();
12337        let client = Client::builder(server.base_url())
12338            .timeout(Duration::from_secs(2))
12339            .build()
12340            .expect("client");
12341
12342        client
12343            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12344            .await
12345            .expect("register");
12346        client
12347            .heartbeat_worker("capture-worker", 1, 1)
12348            .await
12349            .expect("heartbeat");
12350        client
12351            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12352            .await
12353            .expect("workflow poll");
12354        client
12355            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12356            .await
12357            .expect("activity poll");
12358
12359        for path in [
12360            "/api/worker/register",
12361            "/api/worker/heartbeat",
12362            "/api/worker/workflow-tasks/poll",
12363            "/api/worker/activity-tasks/poll",
12364        ] {
12365            assert_eq!(
12366                server.worker_protocol_for(path).as_deref(),
12367                Some(WORKER_PROTOCOL_VERSION),
12368                "unexpected protocol for {path}"
12369            );
12370        }
12371
12372        assert_eq!(
12373            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12374            1
12375        );
12376        assert_eq!(
12377            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12378            1
12379        );
12380        assert!(
12381            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12382                .as_str()
12383                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12384        );
12385        assert!(
12386            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12387                .as_str()
12388                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12389        );
12390    }
12391
12392    #[tokio::test]
12393    async fn query_task_endpoints_send_the_query_feature_protocol() {
12394        let server = MockWorkerServer::start();
12395        let client = Client::builder(server.base_url())
12396            .timeout(Duration::from_secs(2))
12397            .build()
12398            .expect("client");
12399
12400        client
12401            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12402            .await
12403            .expect("query poll");
12404        client
12405            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12406            .await
12407            .expect("query complete");
12408        client
12409            .fail_query_task(
12410                "query-capture",
12411                "capture-worker",
12412                1,
12413                "failed",
12414                "query_rejected",
12415                "QueryFailed",
12416            )
12417            .await
12418            .expect("query fail");
12419
12420        for path in [
12421            "/api/worker/query-tasks/poll",
12422            "/api/worker/query-tasks/query-capture/complete",
12423            "/api/worker/query-tasks/query-capture/fail",
12424        ] {
12425            assert_eq!(
12426                server.worker_protocol_for(path).as_deref(),
12427                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12428                "unexpected protocol for {path}"
12429            );
12430        }
12431
12432        assert_eq!(
12433            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12434            1
12435        );
12436        assert!(
12437            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12438                .as_str()
12439                .is_some_and(|id| id.starts_with("rust-query-poll-"))
12440        );
12441    }
12442
12443    #[tokio::test]
12444    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12445        let server = MockWorkerServer::transient_worker_failures();
12446        let client = Client::builder(server.base_url())
12447            .timeout(Duration::from_secs(2))
12448            .build()
12449            .expect("client");
12450
12451        client
12452            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12453            .await
12454            .expect("workflow poll retry");
12455        client
12456            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12457            .await
12458            .expect("activity poll retry");
12459        client
12460            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12461            .await
12462            .expect("query poll retry");
12463
12464        for path in [
12465            "/api/worker/workflow-tasks/poll",
12466            "/api/worker/activity-tasks/poll",
12467            "/api/worker/query-tasks/poll",
12468        ] {
12469            let bodies = server.request_bodies(path);
12470            assert_eq!(bodies.len(), 2, "{path} must be retried once");
12471            assert_eq!(
12472                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12473                "{path} must preserve the request binding across retry"
12474            );
12475        }
12476    }
12477
12478    #[tokio::test]
12479    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12480        let server = MockWorkerServer::consecutive_poll_failures(2);
12481        let client = Client::builder(server.base_url())
12482            .timeout(Duration::from_secs(2))
12483            .build()
12484            .expect("client");
12485        let mut worker = Worker::new(client, "capture")
12486            .worker_id("capture-worker")
12487            .poll_timeout(Duration::from_millis(10))
12488            .retry_policy(WorkerRetryPolicy {
12489                max_retries: 2,
12490                initial_backoff: Duration::from_millis(1),
12491                max_backoff: Duration::from_millis(1),
12492            });
12493        worker.register_workflow(
12494            "capture.workflow",
12495            |_ctx, _input| async move { Ok(Value::Null) },
12496        );
12497        worker.register_activity(
12498            "capture.activity",
12499            |_ctx, _input| async move { Ok(Value::Null) },
12500        );
12501        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12502            Ok(Value::Null)
12503        });
12504
12505        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12506
12507        for path in [
12508            "/api/worker/workflow-tasks/poll",
12509            "/api/worker/activity-tasks/poll",
12510            "/api/worker/query-tasks/poll",
12511        ] {
12512            let bodies = server.request_bodies(path);
12513            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12514            assert!(
12515                bodies
12516                    .iter()
12517                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12518                "{path} must preserve one request binding across every retry"
12519            );
12520        }
12521    }
12522
12523    #[tokio::test]
12524    async fn query_protocol_rejection_from_older_server_is_typed() {
12525        let server = MockWorkerServer::reject_query_protocol();
12526        let client = Client::builder(server.base_url())
12527            .timeout(Duration::from_secs(2))
12528            .build()
12529            .expect("client");
12530
12531        let error = client
12532            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12533            .await
12534            .expect_err("server below query protocol floor must reject");
12535        let Error::Protocol(failure) = error else {
12536            panic!("expected typed protocol failure");
12537        };
12538
12539        assert_eq!(failure.status, 400);
12540        assert_eq!(failure.reason, "unsupported_protocol_version");
12541        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12542        assert_eq!(
12543            failure.requested_version.as_deref(),
12544            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12545        );
12546        assert_eq!(
12547            server
12548                .worker_protocol_for("/api/worker/query-tasks/poll")
12549                .as_deref(),
12550            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12551        );
12552    }
12553
12554    #[tokio::test]
12555    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12556        let server = MockWorkerServer::reject_query_protocol();
12557        let client = Client::builder(server.base_url())
12558            .timeout(Duration::from_secs(2))
12559            .build()
12560            .expect("client");
12561        let mut worker = Worker::new(client, "rust-workers")
12562            .worker_id("baseline-worker")
12563            .poll_timeout(Duration::from_millis(10));
12564
12565        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12566            Ok(Value::Null)
12567        });
12568
12569        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12570        assert_eq!(
12571            server
12572                .worker_protocol_for("/api/worker/workflow-tasks/poll")
12573                .as_deref(),
12574            Some(WORKER_PROTOCOL_VERSION)
12575        );
12576        assert_eq!(
12577            server.worker_protocol_for("/api/worker/query-tasks/poll"),
12578            None,
12579            "a worker without query handlers must not use the query-task endpoint"
12580        );
12581    }
12582
12583    #[tokio::test]
12584    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12585        let server = MockWorkerServer::reject_query_completion();
12586        let client = Client::builder(server.base_url())
12587            .timeout(Duration::from_secs(2))
12588            .build()
12589            .expect("client");
12590
12591        let error = client
12592            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12593            .await
12594            .expect_err("expired completion must be rejected");
12595        let Error::QueryFailed(failure) = error else {
12596            panic!("expected typed query failure");
12597        };
12598        assert_eq!(failure.status, 409);
12599        assert_eq!(failure.reason, "query_task_timed_out");
12600
12601        let mut worker = Worker::new(client, "rust-workers")
12602            .worker_id("late-worker")
12603            .poll_timeout(Duration::from_millis(10));
12604        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12605        worker.register_query(
12606            "counter",
12607            "current",
12608            |_ctx, _args| async move { Ok(json!(8)) },
12609        );
12610
12611        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12612        assert_eq!(
12613            worker
12614                .run_once()
12615                .await
12616                .expect("worker continues after late completion"),
12617            0
12618        );
12619        assert_eq!(
12620            server.request_count("/api/worker/query-tasks/query-late/complete"),
12621            2
12622        );
12623        assert_eq!(
12624            server.request_count("/api/worker/query-tasks/query-late/fail"),
12625            0,
12626            "a server completion rejection must not be reported as an encoding failure"
12627        );
12628    }
12629
12630    #[tokio::test]
12631    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
12632        let server = MockWorkerServer::start();
12633        let client = Client::builder(server.base_url())
12634            .timeout(Duration::from_secs(2))
12635            .build()
12636            .expect("client");
12637        let mut worker = Worker::new(client, "rust-workers")
12638            .worker_id("activity-only-worker")
12639            .poll_timeout(Duration::from_millis(10));
12640
12641        worker.register_activity(
12642            "activity.only",
12643            |_ctx, _args| async move { Ok(Value::Null) },
12644        );
12645
12646        worker.run_until(async {}).await.expect("run worker");
12647    }
12648
12649    #[tokio::test]
12650    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
12651        let server = MockWorkerServer::start();
12652        let client = Client::builder(server.base_url())
12653            .timeout(Duration::from_secs(2))
12654            .build()
12655            .expect("client");
12656        let mut worker = Worker::new(client, "rust-workers")
12657            .worker_id("workflow-only-worker")
12658            .poll_timeout(Duration::from_millis(10));
12659
12660        worker.register_workflow(
12661            "workflow.only",
12662            |_ctx, _input| async move { Ok(Value::Null) },
12663        );
12664
12665        worker.run_until(async {}).await.expect("run worker");
12666    }
12667
12668    #[tokio::test]
12669    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
12670        let server = MockWorkerServer::start();
12671        let client = Client::builder(server.base_url())
12672            .timeout(Duration::from_secs(2))
12673            .build()
12674            .expect("client");
12675        let observations = Arc::new(Mutex::new(Vec::new()));
12676        let observed = Arc::clone(&observations);
12677        let mut worker = Worker::new(client, "rust-workers")
12678            .worker_id("observed-heartbeat-worker")
12679            .poll_timeout(Duration::from_millis(10))
12680            .on_worker_heartbeat(move |observation| {
12681                observed
12682                    .lock()
12683                    .expect("heartbeat observations")
12684                    .push(observation.clone());
12685            });
12686
12687        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
12688            Ok(Value::Null)
12689        });
12690        let acknowledged = Arc::clone(&observations);
12691        worker
12692            .run_until(async move {
12693                tokio::time::timeout(Duration::from_secs(2), async move {
12694                    loop {
12695                        if !acknowledged
12696                            .lock()
12697                            .expect("heartbeat observations")
12698                            .is_empty()
12699                        {
12700                            break;
12701                        }
12702                        tokio::time::sleep(Duration::from_millis(1)).await;
12703                    }
12704                })
12705                .await
12706                .expect("heartbeat acknowledgement within timeout");
12707            })
12708            .await
12709            .expect("run worker");
12710
12711        let observations = observations.lock().expect("heartbeat observations");
12712        let first = observations.first().expect("heartbeat acknowledgement");
12713        assert_eq!(first.worker_id, "observed-heartbeat-worker");
12714        assert_eq!(first.task_queue, "rust-workers");
12715        assert!(first.acknowledged_at_unix_millis > 0);
12716        assert_eq!(first.acknowledgement, json!({}));
12717    }
12718
12719    #[tokio::test]
12720    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
12721        let server = MockWorkerServer::delayed_heartbeat_worker();
12722        let client = Client::builder(server.base_url())
12723            .timeout(Duration::from_secs(3))
12724            .build()
12725            .expect("client");
12726        let observations = Arc::new(Mutex::new(Vec::new()));
12727        let observed = Arc::clone(&observations);
12728        let mut worker = Worker::new(client, "rust-snapshot-workers")
12729            .worker_id("rust-snapshot-worker")
12730            .poll_timeout(Duration::from_millis(10))
12731            .on_worker_heartbeat(move |observation| {
12732                observed
12733                    .lock()
12734                    .expect("heartbeat observations")
12735                    .push(observation.clone());
12736            });
12737
12738        worker.register_workflow("snapshot", |ctx, _input| async move {
12739            ctx.wait_signal("finish").await?;
12740            Ok(json!({"status": "finished"}))
12741        });
12742        worker.register_query("snapshot", "current", |ctx, _args| async move {
12743            Ok(json!(ctx
12744                .signals("increment")
12745                .iter()
12746                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12747                .sum::<i64>()))
12748        });
12749        worker.register_activity("cancel-aware", |_ctx, _args| async move {
12750            Ok(json!({"late": "completion"}))
12751        });
12752
12753        worker
12754            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
12755            .await
12756            .expect("delayed heartbeat must allow a clean worker shutdown");
12757
12758        let observations = observations.lock().expect("heartbeat observations");
12759        assert!(
12760            observations.len() >= 3,
12761            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
12762        );
12763        assert!(
12764            observations.windows(2).all(|pair| {
12765                pair[1].acknowledged_at_unix_millis
12766                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
12767                    >= 850
12768            }),
12769            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
12770        );
12771        drop(observations);
12772
12773        let heartbeat_times = server.request_times("/api/worker/heartbeat");
12774        let delayed_request_at = *heartbeat_times
12775            .get(1)
12776            .expect("intentionally delayed heartbeat request");
12777        let delay_window_start = delayed_request_at + Duration::from_millis(100);
12778        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
12779        for path in [
12780            "/api/worker/workflow-tasks/poll",
12781            "/api/worker/activity-tasks/poll",
12782            "/api/worker/query-tasks/poll",
12783        ] {
12784            assert!(
12785                server
12786                    .request_times(path)
12787                    .iter()
12788                    .any(|received_at| *received_at >= delay_window_start
12789                        && *received_at <= delay_window_end),
12790                "{path} must keep polling while a heartbeat acknowledgement is delayed"
12791            );
12792        }
12793        assert!(
12794            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
12795            "workflow work must be settled"
12796        );
12797        assert!(
12798            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
12799            "activity work must be settled"
12800        );
12801        assert!(
12802            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
12803            "query work must be settled"
12804        );
12805    }
12806
12807    #[tokio::test]
12808    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
12809        let server = MockWorkerServer::heartbeat_retry_worker();
12810        let client = Client::builder(server.base_url())
12811            .timeout(Duration::from_secs(2))
12812            .build()
12813            .expect("client");
12814        let observations = Arc::new(Mutex::new(Vec::new()));
12815        let observed = Arc::clone(&observations);
12816        let worker = Worker::new(client, "rust-workers")
12817            .worker_id("heartbeat-retry-worker")
12818            .retry_policy(WorkerRetryPolicy {
12819                max_retries: 1,
12820                initial_backoff: Duration::from_millis(300),
12821                max_backoff: Duration::from_millis(300),
12822            })
12823            .on_worker_heartbeat(move |observation| {
12824                observed
12825                    .lock()
12826                    .expect("heartbeat observations")
12827                    .push(observation.clone());
12828            });
12829
12830        worker
12831            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
12832            .await
12833            .expect("retryable heartbeat failure must remain bounded and recover");
12834
12835        let observations = observations.lock().expect("heartbeat observations");
12836        assert!(observations.len() >= 3, "heartbeat retry must recover");
12837        assert!(
12838            observations.windows(2).all(|pair| {
12839                pair[1]
12840                    .acknowledged_at_unix_millis
12841                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
12842                    >= 850
12843            }),
12844            "a successful retry must start a fresh advertised cadence: {observations:?}"
12845        );
12846        assert_eq!(
12847            server.request_count("/api/worker/heartbeat"),
12848            observations.len() + 1,
12849            "one retryable failure must add exactly one bounded request"
12850        );
12851    }
12852
12853    #[tokio::test]
12854    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
12855        let server = MockWorkerServer::waiting_query_worker();
12856        let client = Client::builder(server.base_url())
12857            .timeout(Duration::from_secs(2))
12858            .build()
12859            .expect("client");
12860        let observations = Arc::new(Mutex::new(Vec::new()));
12861        let observed = Arc::clone(&observations);
12862        let mut worker = Worker::new(client, "rust-snapshot-workers")
12863            .worker_id("rust-snapshot-worker")
12864            .poll_timeout(Duration::from_millis(10))
12865            .on_worker_heartbeat(move |observation| {
12866                observed
12867                    .lock()
12868                    .expect("heartbeat observations")
12869                    .push(observation.clone());
12870            });
12871
12872        worker.register_workflow("snapshot", |ctx, _input| async move {
12873            ctx.wait_signal("finish").await?;
12874            Ok(json!({"status": "finished"}))
12875        });
12876        worker.register_query("snapshot", "current", |ctx, _args| async move {
12877            let current = ctx
12878                .signals("increment")
12879                .iter()
12880                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12881                .sum::<i64>();
12882            Ok(json!(current))
12883        });
12884        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
12885
12886        worker
12887            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
12888            .await
12889            .expect("pending workflow and query poller must remain live until shutdown");
12890
12891        assert!(
12892            observations.lock().expect("heartbeat observations").len() >= 4,
12893            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
12894        );
12895        assert!(
12896            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
12897            "workflow polling must continue after empty replay acknowledgements"
12898        );
12899        assert!(
12900            server.request_count("/api/worker/query-tasks/poll") >= 2,
12901            "query polling must continue after serving the current query"
12902        );
12903        assert_eq!(
12904            server.request_body("/api/worker/register")["capabilities"],
12905            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
12906        );
12907        assert_eq!(
12908            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
12909            json!({
12910                "queries": ["current"],
12911                "updates": ["replace"],
12912            })
12913        );
12914
12915        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
12916        assert_eq!(
12917            opened["commands"],
12918            json!([{
12919                "type": "open_signal_wait",
12920                "signal_name": "finish",
12921            }])
12922        );
12923
12924        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
12925            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
12926            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
12927            let failure = server.request_body(&fail_path);
12928            assert_eq!(
12929                failure["failure"]["type"],
12930                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
12931            );
12932            assert_eq!(server.request_count(&completion_path), 0);
12933        }
12934
12935        let query_completion =
12936            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
12937        assert_eq!(query_completion["result"], json!(8));
12938
12939        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
12940        assert_eq!(
12941            server.request_count(terminal_path),
12942            1,
12943            "the matching signal must settle the workflow exactly once"
12944        );
12945        let terminal = server.request_body(terminal_path);
12946        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
12947        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
12948        assert_eq!(
12949            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
12950                .expect("terminal workflow result"),
12951            json!({"status": "finished"})
12952        );
12953    }
12954
12955    #[tokio::test]
12956    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
12957        let server = MockWorkerServer::transient_worker_failures();
12958        let client = Client::builder(server.base_url())
12959            .timeout(Duration::from_secs(2))
12960            .build()
12961            .expect("client");
12962        let mut worker = Worker::new(client, "rust-workers")
12963            .worker_id("retry-worker")
12964            .poll_timeout(Duration::from_millis(10))
12965            .retry_policy(WorkerRetryPolicy {
12966                max_retries: 2,
12967                initial_backoff: Duration::from_millis(1),
12968                max_backoff: Duration::from_millis(1),
12969            });
12970        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12971        worker.register_activity(
12972            "counter.activity",
12973            |_ctx, _input| async move { Ok(Value::Null) },
12974        );
12975        worker.register_query(
12976            "counter",
12977            "current",
12978            |_ctx, _args| async move { Ok(json!(8)) },
12979        );
12980
12981        worker
12982            .run_until(tokio::time::sleep(Duration::from_millis(75)))
12983            .await
12984            .expect("transient failures must not stop the worker");
12985
12986        for path in [
12987            "/api/worker/heartbeat",
12988            "/api/worker/workflow-tasks/poll",
12989            "/api/worker/activity-tasks/poll",
12990            "/api/worker/query-tasks/poll",
12991        ] {
12992            assert!(
12993                server.request_count(path) >= 2,
12994                "{path} must continue after its transient failure"
12995            );
12996        }
12997    }
12998
12999    #[tokio::test]
13000    async fn worker_bounds_transport_retries() {
13001        let server = MockWorkerServer::unavailable_polls();
13002        let client = Client::builder(server.base_url())
13003            .timeout(Duration::from_secs(2))
13004            .build()
13005            .expect("client");
13006        let mut worker = Worker::new(client, "rust-workers")
13007            .worker_id("bounded-retry-worker")
13008            .poll_timeout(Duration::from_millis(10))
13009            .retry_policy(WorkerRetryPolicy {
13010                max_retries: 2,
13011                initial_backoff: Duration::from_millis(1),
13012                max_backoff: Duration::from_millis(1),
13013            });
13014        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13015
13016        let error = worker.run().await.expect_err("retry bound must terminate");
13017        assert!(matches!(error, Error::Transport(_)));
13018        assert_eq!(
13019            server.request_count("/api/worker/workflow-tasks/poll"),
13020            3,
13021            "one initial request plus exactly two retries"
13022        );
13023    }
13024
13025    #[tokio::test]
13026    async fn worker_retry_policy_can_disable_poll_retries() {
13027        let server = MockWorkerServer::unavailable_polls();
13028        let client = Client::builder(server.base_url())
13029            .timeout(Duration::from_secs(2))
13030            .build()
13031            .expect("client");
13032        let mut worker = Worker::new(client, "rust-workers")
13033            .worker_id("no-retry-worker")
13034            .poll_timeout(Duration::from_millis(10))
13035            .retry_policy(WorkerRetryPolicy {
13036                max_retries: 0,
13037                initial_backoff: Duration::from_millis(1),
13038                max_backoff: Duration::from_millis(1),
13039            });
13040        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13041
13042        let error = worker
13043            .run_once()
13044            .await
13045            .expect_err("disabled retries must return the first transport failure");
13046        assert!(matches!(error, Error::Transport(_)));
13047        assert_eq!(
13048            server.request_count("/api/worker/workflow-tasks/poll"),
13049            1,
13050            "max_retries=0 must send only the initial request"
13051        );
13052    }
13053
13054    #[tokio::test]
13055    async fn worker_does_not_retry_authentication_failures() {
13056        let server = MockWorkerServer::unauthorized_polls();
13057        let client = Client::builder(server.base_url())
13058            .timeout(Duration::from_secs(2))
13059            .build()
13060            .expect("client");
13061        let mut worker = Worker::new(client, "rust-workers")
13062            .worker_id("unauthorized-worker")
13063            .poll_timeout(Duration::from_millis(10));
13064        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13065
13066        let error = worker
13067            .run()
13068            .await
13069            .expect_err("authentication must terminate");
13070        let Error::Http { status, body } = error else {
13071            panic!("expected stable HTTP authentication error");
13072        };
13073        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13074        assert!(body.contains("authentication_failed"));
13075        assert_eq!(
13076            server.request_count("/api/worker/workflow-tasks/poll"),
13077            1,
13078            "authentication failures must not be retried"
13079        );
13080    }
13081
13082    #[derive(Clone, Debug)]
13083    struct CapturedRequest {
13084        path: String,
13085        worker_protocol: Option<String>,
13086        body: String,
13087        received_at: Instant,
13088    }
13089
13090    struct MockWorkerServer {
13091        addr: SocketAddr,
13092        stop: Arc<AtomicBool>,
13093        requests: Arc<Mutex<Vec<CapturedRequest>>>,
13094        thread: Option<thread::JoinHandle<()>>,
13095    }
13096
13097    #[derive(Clone, Copy, Default)]
13098    struct MockWorkerBehavior {
13099        reject_query_protocol: bool,
13100        reject_query_completion: bool,
13101        waiting_query_worker: bool,
13102        complete_named_signal: bool,
13103        poll_failures_per_path: usize,
13104        heartbeat_failures: usize,
13105        heartbeat_failure_request: Option<usize>,
13106        delayed_heartbeat_request: Option<usize>,
13107        heartbeat_response_delay: Duration,
13108        concurrent_requests: bool,
13109        unauthorized_polls: bool,
13110        cancelled_activity: bool,
13111        draining_polls: bool,
13112        workflow_completion_status: Option<&'static str>,
13113        workflow_completion_body: Option<&'static str>,
13114    }
13115
13116    impl MockWorkerServer {
13117        fn start() -> Self {
13118            Self::start_with_behavior(MockWorkerBehavior::default())
13119        }
13120
13121        fn reject_query_protocol() -> Self {
13122            Self::start_with_behavior(MockWorkerBehavior {
13123                reject_query_protocol: true,
13124                ..MockWorkerBehavior::default()
13125            })
13126        }
13127
13128        fn reject_query_completion() -> Self {
13129            Self::start_with_behavior(MockWorkerBehavior {
13130                reject_query_completion: true,
13131                ..MockWorkerBehavior::default()
13132            })
13133        }
13134
13135        fn waiting_query_worker() -> Self {
13136            Self::start_with_behavior(MockWorkerBehavior {
13137                waiting_query_worker: true,
13138                complete_named_signal: true,
13139                ..MockWorkerBehavior::default()
13140            })
13141        }
13142
13143        fn transient_worker_failures() -> Self {
13144            Self::start_with_behavior(MockWorkerBehavior {
13145                poll_failures_per_path: 1,
13146                heartbeat_failures: 1,
13147                ..MockWorkerBehavior::default()
13148            })
13149        }
13150
13151        fn consecutive_poll_failures(count: usize) -> Self {
13152            Self::start_with_behavior(MockWorkerBehavior {
13153                poll_failures_per_path: count,
13154                ..MockWorkerBehavior::default()
13155            })
13156        }
13157
13158        fn delayed_heartbeat_worker() -> Self {
13159            Self::start_with_behavior(MockWorkerBehavior {
13160                waiting_query_worker: true,
13161                delayed_heartbeat_request: Some(2),
13162                heartbeat_response_delay: Duration::from_millis(1_500),
13163                concurrent_requests: true,
13164                cancelled_activity: true,
13165                ..MockWorkerBehavior::default()
13166            })
13167        }
13168
13169        fn heartbeat_retry_worker() -> Self {
13170            Self::start_with_behavior(MockWorkerBehavior {
13171                waiting_query_worker: true,
13172                heartbeat_failure_request: Some(2),
13173                concurrent_requests: true,
13174                ..MockWorkerBehavior::default()
13175            })
13176        }
13177
13178        fn unavailable_polls() -> Self {
13179            Self::start_with_behavior(MockWorkerBehavior {
13180                poll_failures_per_path: usize::MAX,
13181                ..MockWorkerBehavior::default()
13182            })
13183        }
13184
13185        fn unauthorized_polls() -> Self {
13186            Self::start_with_behavior(MockWorkerBehavior {
13187                unauthorized_polls: true,
13188                ..MockWorkerBehavior::default()
13189            })
13190        }
13191
13192        fn cancelled_activity() -> Self {
13193            Self::start_with_behavior(MockWorkerBehavior {
13194                cancelled_activity: true,
13195                ..MockWorkerBehavior::default()
13196            })
13197        }
13198
13199        fn draining_polls() -> Self {
13200            Self::start_with_behavior(MockWorkerBehavior {
13201                draining_polls: true,
13202                ..MockWorkerBehavior::default()
13203            })
13204        }
13205
13206        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13207            Self::start_with_behavior(MockWorkerBehavior {
13208                workflow_completion_status: Some(status),
13209                workflow_completion_body: Some(body),
13210                ..MockWorkerBehavior::default()
13211            })
13212        }
13213
13214        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13215            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13216            listener
13217                .set_nonblocking(true)
13218                .expect("configure mock listener");
13219            let addr = listener.local_addr().expect("mock server address");
13220            let stop = Arc::new(AtomicBool::new(false));
13221            let server_stop = Arc::clone(&stop);
13222            let requests = Arc::new(Mutex::new(Vec::new()));
13223            let server_requests = Arc::clone(&requests);
13224            let thread = thread::spawn(move || {
13225                let mut request_threads = Vec::new();
13226                while !server_stop.load(Ordering::SeqCst) {
13227                    match listener.accept() {
13228                        Ok((mut stream, _)) => {
13229                            if behavior.concurrent_requests {
13230                                let requests = Arc::clone(&server_requests);
13231                                request_threads.push(thread::spawn(move || {
13232                                    handle_mock_worker_request(&mut stream, &requests, behavior)
13233                                }));
13234                            } else {
13235                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
13236                            }
13237                        }
13238                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13239                            let mut index = 0;
13240                            while index < request_threads.len() {
13241                                if request_threads[index].is_finished() {
13242                                    request_threads
13243                                        .swap_remove(index)
13244                                        .join()
13245                                        .expect("join mock request");
13246                                } else {
13247                                    index += 1;
13248                                }
13249                            }
13250                            thread::sleep(Duration::from_millis(5));
13251                        }
13252                        Err(_) => break,
13253                    }
13254                }
13255                for request_thread in request_threads {
13256                    request_thread.join().expect("join mock request");
13257                }
13258            });
13259
13260            Self {
13261                addr,
13262                stop,
13263                requests,
13264                thread: Some(thread),
13265            }
13266        }
13267
13268        fn base_url(&self) -> String {
13269            format!("http://{}", self.addr)
13270        }
13271
13272        fn worker_protocol_for(&self, path: &str) -> Option<String> {
13273            self.requests
13274                .lock()
13275                .expect("captured requests")
13276                .iter()
13277                .find(|request| request.path == path)
13278                .and_then(|request| request.worker_protocol.clone())
13279        }
13280
13281        fn request_count(&self, path: &str) -> usize {
13282            self.requests
13283                .lock()
13284                .expect("captured requests")
13285                .iter()
13286                .filter(|request| request.path == path)
13287                .count()
13288        }
13289
13290        fn request_times(&self, path: &str) -> Vec<Instant> {
13291            self.requests
13292                .lock()
13293                .expect("captured requests")
13294                .iter()
13295                .filter(|request| request.path == path)
13296                .map(|request| request.received_at)
13297                .collect()
13298        }
13299
13300        fn request_body(&self, path: &str) -> Value {
13301            let requests = self.requests.lock().expect("captured requests");
13302            let body = &requests
13303                .iter()
13304                .find(|request| request.path == path)
13305                .unwrap_or_else(|| panic!("missing request for {path}"))
13306                .body;
13307            serde_json::from_str(body).unwrap_or_else(|error| {
13308                panic!("invalid JSON request body for {path}: {error}: {body:?}")
13309            })
13310        }
13311
13312        fn request_bodies(&self, path: &str) -> Vec<Value> {
13313            self.requests
13314                .lock()
13315                .expect("captured requests")
13316                .iter()
13317                .filter(|request| request.path == path)
13318                .map(|request| {
13319                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
13320                        panic!(
13321                            "invalid JSON request body for {path}: {error}: {:?}",
13322                            request.body
13323                        )
13324                    })
13325                })
13326                .collect()
13327        }
13328    }
13329
13330    impl Drop for MockWorkerServer {
13331        fn drop(&mut self) {
13332            self.stop.store(true, Ordering::SeqCst);
13333            let _ = TcpStream::connect(self.addr);
13334
13335            if let Some(thread) = self.thread.take() {
13336                thread.join().expect("join mock server");
13337            }
13338        }
13339    }
13340
13341    fn handle_mock_worker_request(
13342        stream: &mut TcpStream,
13343        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13344        behavior: MockWorkerBehavior,
13345    ) {
13346        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13347        let mut buffer = [0_u8; 8192];
13348        let mut request = Vec::new();
13349
13350        loop {
13351            match stream.read(&mut buffer) {
13352                Ok(0) => break,
13353                Ok(read) => {
13354                    request.extend_from_slice(&buffer[..read]);
13355                    if mock_request_is_complete(&request) {
13356                        break;
13357                    }
13358                }
13359                Err(error)
13360                    if matches!(
13361                        error.kind(),
13362                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13363                    ) =>
13364                {
13365                    break;
13366                }
13367                Err(_) => return,
13368            }
13369        }
13370
13371        let request = String::from_utf8_lossy(&request);
13372        let body = request
13373            .split_once("\r\n\r\n")
13374            .map(|(_, body)| body)
13375            .unwrap_or_default();
13376        let path = request
13377            .lines()
13378            .next()
13379            .and_then(|line| line.split_whitespace().nth(1))
13380            .unwrap_or_default();
13381        let worker_protocol = request.lines().find_map(|line| {
13382            let (name, value) = line.split_once(':')?;
13383            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13384                .then(|| value.trim().to_string())
13385        });
13386        let request_number = {
13387            let mut requests = requests.lock().expect("captured requests");
13388            requests.push(CapturedRequest {
13389                path: path.to_string(),
13390                worker_protocol: worker_protocol.clone(),
13391                body: body.to_string(),
13392                received_at: Instant::now(),
13393            });
13394            requests
13395                .iter()
13396                .filter(|request| request.path == path)
13397                .count()
13398        };
13399
13400        let is_poll = matches!(
13401            path,
13402            "/api/worker/workflow-tasks/poll"
13403                | "/api/worker/activity-tasks/poll"
13404                | "/api/worker/query-tasks/poll"
13405        );
13406        if is_poll && request_number <= behavior.poll_failures_per_path {
13407            return;
13408        }
13409        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
13410            return;
13411        }
13412        if path == "/api/worker/heartbeat"
13413            && behavior.heartbeat_failure_request == Some(request_number)
13414        {
13415            return;
13416        }
13417        if path == "/api/worker/heartbeat"
13418            && behavior.delayed_heartbeat_request == Some(request_number)
13419        {
13420            thread::sleep(behavior.heartbeat_response_delay);
13421        }
13422        if behavior.unauthorized_polls && is_poll {
13423            write_mock_response(
13424                stream,
13425                "401 Unauthorized",
13426                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
13427            );
13428            return;
13429        }
13430        if behavior.draining_polls && is_poll {
13431            write_mock_response(
13432                stream,
13433                "409 Conflict",
13434                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
13435            );
13436            return;
13437        }
13438
13439        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
13440            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
13441            let body = format!(
13442                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
13443            );
13444            write_mock_response(stream, "400 Bad Request", &body);
13445            return;
13446        }
13447
13448        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
13449        {
13450            write_mock_response(
13451                stream,
13452                "409 Conflict",
13453                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
13454            );
13455            return;
13456        }
13457
13458        if behavior.workflow_completion_status.is_some()
13459            && path == "/api/worker/workflow-tasks/poll"
13460            && request_number == 1
13461        {
13462            write_mock_response(
13463                stream,
13464                "200 OK",
13465                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
13466            );
13467            return;
13468        }
13469
13470        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
13471            if let (Some(status), Some(body)) = (
13472                behavior.workflow_completion_status,
13473                behavior.workflow_completion_body,
13474            ) {
13475                write_mock_response(stream, status, body);
13476                return;
13477            }
13478        }
13479
13480        if behavior.waiting_query_worker {
13481            if behavior.complete_named_signal
13482                && path == "/api/worker/workflow-tasks/poll"
13483                && request_number == 1
13484            {
13485                let body = json!({
13486                    "task": {
13487                        "task_id": "snapshot-open",
13488                        "workflow_id": "snapshot-1",
13489                        "run_id": "snapshot-run-1",
13490                        "workflow_type": "snapshot",
13491                        "payload_codec": DEFAULT_CODEC,
13492                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13493                            .expect("Avro workflow arguments"),
13494                        "history_events": [],
13495                        "workflow_task_attempt": 1,
13496                        "lease_owner": "rust-snapshot-worker"
13497                    }
13498                })
13499                .to_string();
13500                write_mock_response(stream, "200 OK", &body);
13501                return;
13502            }
13503
13504            let signal_request = request_number - usize::from(behavior.complete_named_signal);
13505            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
13506            if path == "/api/worker/workflow-tasks/poll"
13507                && signal_request >= 1
13508                && signal_request <= signal_request_limit
13509            {
13510                let finish = behavior.complete_named_signal && signal_request == 3;
13511                let amounts = if signal_request == 1 {
13512                    vec![3]
13513                } else {
13514                    vec![3, 5]
13515                };
13516                let task_id = if signal_request == 1 {
13517                    "snapshot-wait-3"
13518                } else if finish {
13519                    "snapshot-finish"
13520                } else {
13521                    "snapshot-wait-5"
13522                };
13523                let mut history_events = std::iter::once(json!({
13524                    "event_type": "SignalWaitOpened",
13525                    "payload": {"sequence": 1, "signal_name": "finish"}
13526                }))
13527                .chain(amounts.iter().enumerate().map(|(index, amount)| {
13528                    json!({
13529                        "event_type": "SignalReceived",
13530                        "payload": {
13531                            "signal_id": format!("increment-{amount}"),
13532                            "signal_name": "increment",
13533                            "workflow_sequence": index + 2,
13534                            "payload_codec": DEFAULT_CODEC,
13535                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13536                                .expect("Avro signal envelope")
13537                        }
13538                    })
13539                }))
13540                .collect::<Vec<_>>();
13541                let (resume_id, resume_name, resume_arguments) = if finish {
13542                    history_events.push(json!({
13543                        "event_type": "SignalReceived",
13544                        "payload": {
13545                            "signal_id": "finish",
13546                            "signal_name": "finish",
13547                            "workflow_sequence": 4,
13548                            "payload_codec": DEFAULT_CODEC,
13549                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13550                                .expect("Avro finish signal envelope")
13551                        }
13552                    }));
13553                    (
13554                        "finish".to_string(),
13555                        "finish".to_string(),
13556                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
13557                            .expect("Avro finish resume signal"),
13558                    )
13559                } else {
13560                    let amount = amounts.last().expect("amount");
13561                    (
13562                        format!("increment-{amount}"),
13563                        "increment".to_string(),
13564                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13565                            .expect("Avro increment resume signal"),
13566                    )
13567                };
13568                let body = json!({
13569                    "task": {
13570                        "task_id": task_id,
13571                        "workflow_id": "snapshot-1",
13572                        "run_id": "snapshot-run-1",
13573                        "workflow_type": "snapshot",
13574                        "payload_codec": DEFAULT_CODEC,
13575                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13576                            .expect("Avro workflow arguments"),
13577                        "history_events": history_events,
13578                        "workflow_task_attempt": 1,
13579                        "workflow_signal_id": resume_id,
13580                        "signal_name": resume_name,
13581                        "signal_arguments": resume_arguments,
13582                        "lease_owner": "rust-snapshot-worker"
13583                    }
13584                })
13585                .to_string();
13586                write_mock_response(stream, "200 OK", &body);
13587                return;
13588            }
13589
13590            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
13591                let history_events = [3, 5]
13592                    .into_iter()
13593                    .enumerate()
13594                    .map(|(index, amount)| {
13595                        json!({
13596                            "event_type": "SignalReceived",
13597                            "payload": {
13598                                "signal_id": format!("increment-{amount}"),
13599                                "signal_name": "increment",
13600                                "workflow_sequence": index + 2,
13601                                "payload_codec": DEFAULT_CODEC,
13602                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13603                                    .expect("Avro query signal envelope")
13604                            }
13605                        })
13606                    })
13607                    .collect::<Vec<_>>();
13608                let body = json!({
13609                    "task": {
13610                        "query_task_id": "snapshot-current",
13611                        "query_task_attempt": 1,
13612                        "lease_owner": "rust-snapshot-worker",
13613                        "workflow_id": "snapshot-1",
13614                        "run_id": "snapshot-run-1",
13615                        "workflow_type": "snapshot",
13616                        "query_name": "current",
13617                        "payload_codec": DEFAULT_CODEC,
13618                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13619                            .expect("Avro workflow arguments"),
13620                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13621                            .expect("Avro query arguments"),
13622                        "history_events": history_events,
13623                        "run_status": "waiting"
13624                    }
13625                })
13626                .to_string();
13627                write_mock_response(stream, "200 OK", &body);
13628                return;
13629            }
13630
13631            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
13632                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
13633            {
13634                write_mock_response(
13635                    stream,
13636                    "200 OK",
13637                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
13638                );
13639                return;
13640            }
13641
13642            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
13643                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
13644                return;
13645            }
13646
13647            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
13648                write_mock_response(
13649                    stream,
13650                    "200 OK",
13651                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
13652                );
13653                return;
13654            }
13655
13656            if path == "/api/worker/query-tasks/snapshot-current/complete" {
13657                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
13658                return;
13659            }
13660        }
13661
13662        if matches!(
13663            path,
13664            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
13665        ) {
13666            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13667                .expect("typed mock result");
13668            let body = json!({
13669                "result": typed_fidelity_probe().into_json().expect("result projection"),
13670                "result_envelope": result,
13671            })
13672            .to_string();
13673            write_mock_response(stream, "200 OK", &body);
13674            return;
13675        }
13676
13677        if path == "/api/workflows/typed-1" {
13678            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13679                .expect("typed mock result");
13680            let body = json!({
13681                "workflow_id": "typed-1",
13682                "run_id": "run-typed-1",
13683                "workflow_type": "typed.echo",
13684                "status": "completed",
13685                "output": typed_fidelity_probe().into_json().expect("output projection"),
13686                "output_envelope": result,
13687            })
13688            .to_string();
13689            write_mock_response(stream, "200 OK", &body);
13690            return;
13691        }
13692
13693        let (status, body) = match path {
13694            "/api/workflows" => (
13695                "201 Created",
13696                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
13697            ),
13698            "/api/worker/register" if behavior.waiting_query_worker => (
13699                "200 OK",
13700                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
13701            ),
13702            "/api/worker/register" => (
13703                "200 OK",
13704                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
13705            ),
13706            "/api/worker/heartbeat" => ("200 OK", "{}"),
13707            "/api/worker/activity-tasks/poll"
13708                if behavior.cancelled_activity && request_number == 1 =>
13709            {
13710                (
13711                    "200 OK",
13712                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
13713                )
13714            }
13715            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
13716                ("200 OK", r#"{"task":null}"#)
13717            }
13718            "/api/worker/query-tasks/poll"
13719                if behavior.reject_query_completion && request_number == 1 =>
13720            {
13721                (
13722                    "200 OK",
13723                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
13724                )
13725            }
13726            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
13727            "/api/worker/query-tasks/query-capture/complete"
13728            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
13729            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
13730                "200 OK",
13731                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
13732            ),
13733            "/api/worker/activity-tasks/activity-cancel/complete" => (
13734                "409 Conflict",
13735                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
13736            ),
13737            "/api/worker/activity-tasks/activity-typed/complete"
13738            | "/api/worker/activity-tasks/activity-typed/fail"
13739            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
13740            "/api/workflows/counter-1/query/current" => (
13741                "200 OK",
13742                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
13743            ),
13744            "/api/workflows/counter-1/query/missing" => (
13745                "404 Not Found",
13746                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
13747            ),
13748            "/api/workflows/wf-lifecycle/cancel" => (
13749                "200 OK",
13750                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
13751            ),
13752            "/api/workflows/wf-lifecycle/terminate" => (
13753                "200 OK",
13754                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
13755            ),
13756            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
13757                "200 OK",
13758                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
13759            ),
13760            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
13761                "200 OK",
13762                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
13763            ),
13764            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
13765            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
13766                "409 Conflict",
13767                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
13768            ),
13769            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
13770                "200 OK",
13771                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
13772            ),
13773            "/api/workflows/wf-cancelled" => (
13774                "200 OK",
13775                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
13776            ),
13777            "/api/workflows/wf-terminated" => (
13778                "200 OK",
13779                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
13780            ),
13781            "/api/workflows/wf-timed-out" => (
13782                "200 OK",
13783                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
13784            ),
13785            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
13786                "200 OK",
13787                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
13788            ),
13789            "/api/workflows/wf-selected" => (
13790                "200 OK",
13791                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
13792            ),
13793            "/api/workflows/wf-selected/runs/run-selected" => (
13794                "200 OK",
13795                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
13796            ),
13797            _ => ("404 Not Found", r#"{"message":"not found"}"#),
13798        };
13799        write_mock_response(stream, status, body);
13800    }
13801
13802    fn mock_request_is_complete(request: &[u8]) -> bool {
13803        let Some(header_end) = request
13804            .windows(4)
13805            .position(|window| window == b"\r\n\r\n")
13806            .map(|position| position + 4)
13807        else {
13808            return false;
13809        };
13810        let headers = String::from_utf8_lossy(&request[..header_end]);
13811        let content_length = headers.lines().find_map(|line| {
13812            let (name, value) = line.split_once(':')?;
13813            name.eq_ignore_ascii_case("content-length")
13814                .then(|| value.trim().parse::<usize>().ok())
13815                .flatten()
13816        });
13817
13818        request.len() >= header_end + content_length.unwrap_or(0)
13819    }
13820
13821    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
13822        let response = format!(
13823            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
13824            body.len()
13825        );
13826
13827        let _ = stream.write_all(response.as_bytes());
13828        let _ = stream.flush();
13829    }
13830}