Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21    de::DeserializeOwned,
22    ser::{SerializeMap, SerializeSeq},
23    Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
33/// Worker-registration capability for server-routed read-only queries.
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
35/// Worker-registration capability for synchronous workflow updates.
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
37/// First additive worker protocol that defines query-task transport.
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
39
40const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
41const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
42    "Workflow task waiting for scheduled history.";
43const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
44
45const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
46    "lease_expired",
47    "query_task_not_found",
48    "query_task_not_leased",
49    "query_task_timed_out",
50];
51
52/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
53pub const AVRO_VALUE_SCHEMA_JSON: &str =
54    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
55pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
57const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
58
59static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
60
61#[derive(Clone, Copy)]
62enum RequestProtocol {
63    ControlPlane,
64    Worker(&'static str),
65}
66
67pub type Result<T> = std::result::Result<T, Error>;
68
69#[derive(Debug, Error)]
70pub enum Error {
71    #[error("transport error: {0}")]
72    Transport(#[from] reqwest::Error),
73    #[error(
74        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
75    )]
76    InvalidBaseUrl,
77    #[error("json error: {0}")]
78    Json(#[from] serde_json::Error),
79    #[error("http {status}: {body}")]
80    Http {
81        status: reqwest::StatusCode,
82        body: String,
83    },
84    #[error("codec error: {0}")]
85    Codec(String),
86    #[error(transparent)]
87    QueryFailed(QueryFailure),
88    #[error(transparent)]
89    Protocol(ProtocolFailure),
90    #[error(transparent)]
91    NonDeterministicReplay(ReplayFailure),
92    #[error(transparent)]
93    ChildWorkflowFailed(ChildWorkflowFailure),
94    #[error(transparent)]
95    ActivityFailed(ActivityFailure),
96    #[error(transparent)]
97    WorkflowCommandRejected(WorkflowCommandRejection),
98    #[error(transparent)]
99    WorkflowFailed(WorkflowTerminalOutcome),
100    #[error(transparent)]
101    WorkflowCancelled(WorkflowTerminalOutcome),
102    #[error(transparent)]
103    WorkflowTerminated(WorkflowTerminalOutcome),
104    #[error(transparent)]
105    WorkflowTimedOut(WorkflowTerminalOutcome),
106    #[error(transparent)]
107    ActivityTaskRejected(ActivityTaskRejection),
108    #[error("workflow handler {0:?} is not registered")]
109    WorkflowNotRegistered(String),
110    #[error("activity handler {0:?} is not registered")]
111    ActivityNotRegistered(String),
112    #[error("workflow future yielded without emitting a durable command")]
113    WorkflowYieldedWithoutCommand,
114    #[error("workflow state lock is poisoned")]
115    WorkflowStatePoisoned,
116    #[error("timer duration is too large for the worker protocol")]
117    TimerDurationOverflow,
118    #[error("operation timed out")]
119    Timeout,
120    #[error(
121        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
122    )]
123    MissingRoleCredentials {
124        role: &'static str,
125        opposite_role: &'static str,
126    },
127    #[error("worker loop error: {0}")]
128    WorkerLoop(String),
129    #[error(
130        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
131    )]
132    UnsupportedUpdateValidators { workflow_type: String },
133    #[error("{primary}; worker deregistration also failed: {deregistration}")]
134    WorkerShutdown {
135        primary: Box<Error>,
136        deregistration: Box<Error>,
137    },
138    #[error("invalid child workflow options: {0}")]
139    InvalidChildWorkflowOptions(String),
140    #[error(transparent)]
141    InvalidActivityOptions(ActivityOptionsError),
142    #[error(transparent)]
143    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
144    #[doc(hidden)]
145    #[error("workflow requested continue as new")]
146    ContinueAsNew(ContinueAsNewRequest),
147}
148
149/// The lifecycle command sent to a workflow execution.
150#[derive(Clone, Copy, Debug, PartialEq, Eq)]
151pub enum WorkflowCommandKind {
152    Cancel,
153    Terminate,
154}
155
156impl WorkflowCommandKind {
157    fn as_str(self) -> &'static str {
158        match self {
159            Self::Cancel => "cancel",
160            Self::Terminate => "terminate",
161        }
162    }
163}
164
165/// Optional structured fields for a cancellation or termination request.
166#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
167pub struct WorkflowCommandOptions {
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub reason: Option<String>,
170    #[serde(skip_serializing_if = "Option::is_none")]
171    pub request_id: Option<String>,
172}
173
174/// Server-enforced timeout policy for a workflow start.
175///
176/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
177/// only bounds how long the caller waits. A server deadline produces a terminal
178/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
179/// `run_timeout`.
180#[derive(Clone, Debug, PartialEq, Eq)]
181pub struct WorkflowStartOptions {
182    pub execution_timeout_seconds: u64,
183    pub run_timeout_seconds: u64,
184}
185
186impl Default for WorkflowStartOptions {
187    fn default() -> Self {
188        Self {
189            execution_timeout_seconds: 3600,
190            run_timeout_seconds: 600,
191        }
192    }
193}
194
195impl WorkflowStartOptions {
196    pub fn new() -> Self {
197        Self::default()
198    }
199
200    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
201        self.execution_timeout_seconds = seconds;
202        self
203    }
204
205    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
206        self.run_timeout_seconds = seconds;
207        self
208    }
209
210    fn validate(&self) -> Result<()> {
211        if self.execution_timeout_seconds == 0 {
212            return Err(Error::Codec(
213                "execution_timeout_seconds must be at least 1".to_string(),
214            ));
215        }
216        if self.run_timeout_seconds == 0 {
217            return Err(Error::Codec(
218                "run_timeout_seconds must be at least 1".to_string(),
219            ));
220        }
221        if self.run_timeout_seconds > self.execution_timeout_seconds {
222            return Err(Error::Codec(
223                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
224            ));
225        }
226
227        Ok(())
228    }
229}
230
231/// Optional routing overrides for a continue-as-new transition.
232///
233/// Omitted values retain the current workflow type and task queue. Server-owned
234/// instance metadata is not accepted here and is carried by the server.
235#[derive(Clone, Debug, Default, PartialEq, Eq)]
236pub struct ContinueAsNewOptions {
237    pub workflow_type: Option<String>,
238    pub task_queue: Option<String>,
239}
240
241impl ContinueAsNewOptions {
242    pub fn new() -> Self {
243        Self::default()
244    }
245
246    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
247        self.workflow_type = Some(workflow_type.into());
248        self
249    }
250
251    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
252        self.task_queue = Some(task_queue.into());
253        self
254    }
255
256    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
257        for (field, value) in [
258            ("workflow_type", self.workflow_type.as_deref()),
259            ("task_queue", self.task_queue.as_deref()),
260        ] {
261            if value.is_some_and(|value| value.trim().is_empty()) {
262                return Err(ContinueAsNewOptionsError {
263                    field,
264                    message: format!("{field} must not be empty"),
265                });
266            }
267        }
268        Ok(())
269    }
270}
271
272/// A stable validation error raised before a continue-as-new command is emitted.
273#[derive(Clone, Debug, Error, PartialEq, Eq)]
274#[error("invalid continue-as-new option {field}: {message}")]
275pub struct ContinueAsNewOptionsError {
276    pub field: &'static str,
277    pub message: String,
278}
279
280/// Public history-budget information attached to the current workflow task.
281#[derive(Clone, Debug, Default, PartialEq, Eq)]
282pub struct WorkflowHistoryBudget {
283    pub event_count: u64,
284    pub size_bytes: Option<u64>,
285    pub continue_as_new_recommended: bool,
286    pub pressure: Option<String>,
287}
288
289#[doc(hidden)]
290#[derive(Clone, Debug)]
291pub struct ContinueAsNewRequest {
292    arguments: AvroValue,
293    options: ContinueAsNewOptions,
294}
295
296impl WorkflowCommandOptions {
297    pub fn new() -> Self {
298        Self::default()
299    }
300
301    pub fn reason(mut self, reason: impl Into<String>) -> Self {
302        self.reason = Some(reason.into());
303        self
304    }
305
306    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
307        self.request_id = Some(request_id.into());
308        self
309    }
310}
311
312/// The accepted, machine-readable result of a lifecycle command.
313#[derive(Clone, Debug, PartialEq)]
314pub struct WorkflowCommandResult {
315    pub command: WorkflowCommandKind,
316    pub workflow_id: String,
317    pub run_id: Option<String>,
318    pub outcome: Option<String>,
319    pub reason: Option<String>,
320    pub command_status: Option<String>,
321    pub raw: Value,
322}
323
324/// A stable rejection returned by instance- or selected-run lifecycle commands.
325#[derive(Clone, Debug, Error)]
326#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
327pub struct WorkflowCommandRejection {
328    pub command: WorkflowCommandKind,
329    pub status: u16,
330    pub reason: String,
331    pub message: String,
332    pub workflow_id: String,
333    pub run_id: Option<String>,
334    pub target_scope: Option<String>,
335    pub body: Value,
336}
337
338/// Stable terminal categories returned by [`WorkflowHandle::result`].
339#[derive(Clone, Copy, Debug, PartialEq, Eq)]
340pub enum WorkflowTerminalKind {
341    Failed,
342    Cancelled,
343    Terminated,
344    TimedOut,
345}
346
347/// A typed terminal workflow outcome with durable identity and failure metadata.
348///
349/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
350/// of parsing its display representation. Fields remain `None` when an older
351/// server did not publish that metadata.
352#[derive(Clone, Debug, Error)]
353#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
354pub struct WorkflowTerminalOutcome {
355    pub kind: WorkflowTerminalKind,
356    pub workflow_id: String,
357    pub run_id: Option<String>,
358    pub reason: String,
359    pub failure_category: Option<String>,
360    pub failure_id: Option<String>,
361    pub exception_type: Option<String>,
362    pub exception_class: Option<String>,
363    pub non_retryable: Option<bool>,
364    pub message: Option<String>,
365    pub exception: Option<Value>,
366    pub raw: Value,
367}
368
369/// A worker-side activity settlement or heartbeat rejected by durable state.
370#[derive(Clone, Debug, Error)]
371#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
372pub struct ActivityTaskRejection {
373    pub operation: String,
374    pub status: u16,
375    pub reason: String,
376    pub task_id: String,
377    pub activity_attempt_id: String,
378    pub cancel_requested: bool,
379    pub can_continue: Option<bool>,
380    pub run_closed_reason: Option<String>,
381    pub body: Value,
382}
383
384/// Stable validation categories for [`ActivityOptions`].
385#[derive(Clone, Copy, Debug, PartialEq, Eq)]
386pub enum ActivityOptionsErrorKind {
387    EmptyTaskQueue,
388    EmptyRetryPolicy,
389    InvalidMaxAttempts,
390    BackoffWithoutRetryBudget,
391    TooManyBackoffIntervals,
392    InvalidBackoffCoefficient,
393    BackoffGenerationTooLarge,
394    BackoffOverflow,
395    EmptyNonRetryableErrorType,
396    TimeoutNotPositive,
397    TimeoutOverflow,
398    TimeoutOrder,
399}
400
401/// A machine-readable activity-options validation failure.
402#[derive(Clone, Debug, Error, PartialEq, Eq)]
403#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
404pub struct ActivityOptionsError {
405    pub kind: ActivityOptionsErrorKind,
406    pub field: Option<&'static str>,
407    pub message: String,
408}
409
410impl ActivityOptionsError {
411    fn new(
412        kind: ActivityOptionsErrorKind,
413        field: Option<&'static str>,
414        message: impl Into<String>,
415    ) -> Self {
416        Self {
417            kind,
418            field,
419            message: message.into(),
420        }
421    }
422}
423
424/// Stable terminal categories returned when an awaited activity does not succeed.
425#[derive(Clone, Copy, Debug, PartialEq, Eq)]
426pub enum ActivityFailureKind {
427    Failed,
428    Cancelled,
429    TimedOut,
430}
431
432/// A stable, machine-readable terminal activity failure.
433///
434/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
435/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
436#[derive(Clone, Debug, Error)]
437#[error("activity failed ({reason}): {message}")]
438pub struct ActivityFailure {
439    pub kind: ActivityFailureKind,
440    pub reason: String,
441    pub message: String,
442    pub activity_execution_id: Option<String>,
443    pub activity_attempt_id: Option<String>,
444    pub activity_type: Option<String>,
445    pub activity_class: Option<String>,
446    pub attempt_number: Option<u64>,
447    pub failure_id: Option<String>,
448    pub failure_category: Option<String>,
449    pub timeout_kind: Option<String>,
450    pub non_retryable: bool,
451    pub exception_type: Option<String>,
452    pub exception_class: Option<String>,
453    pub code: Option<Value>,
454    pub exception: Option<Value>,
455}
456
457/// Stable terminal categories returned when an awaited child does not succeed.
458#[derive(Clone, Copy, Debug, PartialEq, Eq)]
459pub enum ChildWorkflowFailureKind {
460    Failed,
461    Cancelled,
462    Terminated,
463}
464
465/// A stable, machine-readable child workflow failure delivered to its parent.
466///
467/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
468/// of parsing the display message. Child and parent identifiers retain the
469/// relationship recorded in durable history across worker restarts.
470#[derive(Clone, Debug, Error)]
471#[error("child workflow failed ({reason}): {message}")]
472pub struct ChildWorkflowFailure {
473    pub kind: ChildWorkflowFailureKind,
474    pub reason: String,
475    pub message: String,
476    pub parent_workflow_id: Option<String>,
477    pub parent_workflow_run_id: Option<String>,
478    pub child_workflow_id: Option<String>,
479    pub child_workflow_run_id: Option<String>,
480    pub child_workflow_type: Option<String>,
481    pub failure_id: Option<String>,
482    pub failure_category: Option<String>,
483    pub exception_type: Option<String>,
484    pub exception_class: Option<String>,
485    pub non_retryable: bool,
486    pub code: Option<Value>,
487    pub exception: Option<Value>,
488}
489
490/// The identity of one durable workflow execution.
491#[derive(Clone, Debug, PartialEq, Eq)]
492pub struct WorkflowIdentity {
493    pub workflow_id: Option<String>,
494    pub run_id: Option<String>,
495}
496
497/// A successful child result together with its durable parent-child identity.
498#[derive(Clone, Debug, PartialEq)]
499pub struct ChildWorkflowResult {
500    pub parent: WorkflowIdentity,
501    pub child: WorkflowIdentity,
502    pub child_workflow_type: Option<String>,
503    pub result: Value,
504}
505
506/// Lossless successful child result for fixed Avro Value workflows.
507#[derive(Clone, Debug, PartialEq)]
508pub struct ChildWorkflowAvroResult {
509    pub parent: WorkflowIdentity,
510    pub child: WorkflowIdentity,
511    pub child_workflow_type: Option<String>,
512    pub result: AvroValue,
513}
514
515/// Server behavior when a parent closes while its child is still open.
516#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
517pub enum ParentClosePolicy {
518    #[default]
519    Abandon,
520    RequestCancel,
521    Terminate,
522}
523
524impl ParentClosePolicy {
525    fn as_str(self) -> &'static str {
526        match self {
527            Self::Abandon => "abandon",
528            Self::RequestCancel => "request_cancel",
529            Self::Terminate => "terminate",
530        }
531    }
532}
533
534/// Durable retry policy for one child workflow invocation.
535#[derive(Clone, Debug, Default, PartialEq, Eq)]
536pub struct ChildWorkflowRetryPolicy {
537    pub max_attempts: Option<u32>,
538    pub backoff_seconds: Vec<u64>,
539    pub non_retryable_error_types: Vec<String>,
540}
541
542/// Options recorded with a child-workflow command.
543///
544/// The task queue is mandatory so routing is explicit and replay-stable.
545#[derive(Clone, Debug, PartialEq, Eq)]
546pub struct ChildWorkflowOptions {
547    pub task_queue: String,
548    pub parent_close_policy: ParentClosePolicy,
549    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
550    pub execution_timeout_seconds: Option<u64>,
551    pub run_timeout_seconds: Option<u64>,
552}
553
554impl ChildWorkflowOptions {
555    pub fn new(task_queue: impl Into<String>) -> Self {
556        Self {
557            task_queue: task_queue.into(),
558            parent_close_policy: ParentClosePolicy::Abandon,
559            retry_policy: None,
560            execution_timeout_seconds: None,
561            run_timeout_seconds: None,
562        }
563    }
564
565    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
566        self.parent_close_policy = policy;
567        self
568    }
569
570    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
571        self.retry_policy = Some(policy);
572        self
573    }
574
575    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
576        self.execution_timeout_seconds = Some(seconds);
577        self
578    }
579
580    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
581        self.run_timeout_seconds = Some(seconds);
582        self
583    }
584}
585
586/// Backoff intervals for one durable activity retry policy.
587#[derive(Clone, Debug, PartialEq, Eq)]
588pub enum ActivityBackoff {
589    /// Use these intervals between attempts. The server repeats the final
590    /// interval if the retry budget contains more attempts than entries.
591    Explicit(Vec<Duration>),
592    /// Generate one interval for every retry using integer exponential growth.
593    Exponential {
594        initial_interval: Duration,
595        coefficient: u32,
596        maximum_interval: Option<Duration>,
597    },
598}
599
600/// Durable server-side retry policy for one activity execution.
601#[derive(Clone, Debug, Default, PartialEq, Eq)]
602pub struct ActivityRetryPolicy {
603    pub max_attempts: Option<u32>,
604    pub backoff: Option<ActivityBackoff>,
605    pub non_retryable_error_types: Vec<String>,
606}
607
608impl ActivityRetryPolicy {
609    /// Start a policy with a finite attempt budget, including the first attempt.
610    pub fn new(max_attempts: u32) -> Self {
611        Self {
612            max_attempts: Some(max_attempts),
613            ..Self::default()
614        }
615    }
616
617    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
618        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
619        self
620    }
621
622    pub fn exponential_backoff(
623        mut self,
624        initial_interval: Duration,
625        coefficient: u32,
626        maximum_interval: Option<Duration>,
627    ) -> Self {
628        self.backoff = Some(ActivityBackoff::Exponential {
629            initial_interval,
630            coefficient,
631            maximum_interval,
632        });
633        self
634    }
635
636    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
637        self.non_retryable_error_types.push(error_type.into());
638        self
639    }
640
641    pub fn non_retryable_error_types(
642        mut self,
643        error_types: impl IntoIterator<Item = impl Into<String>>,
644    ) -> Self {
645        self.non_retryable_error_types
646            .extend(error_types.into_iter().map(Into::into));
647        self
648    }
649}
650
651/// Options recorded atomically on one deterministic `schedule_activity` command.
652///
653/// Durations are rounded up to whole seconds when encoded, so the server never
654/// receives a shorter timeout or backoff than the caller requested.
655#[derive(Clone, Debug, Default, PartialEq, Eq)]
656pub struct ActivityOptions {
657    pub task_queue: Option<String>,
658    pub retry_policy: Option<ActivityRetryPolicy>,
659    pub start_to_close_timeout: Option<Duration>,
660    pub schedule_to_start_timeout: Option<Duration>,
661    pub schedule_to_close_timeout: Option<Duration>,
662    pub heartbeat_timeout: Option<Duration>,
663}
664
665impl ActivityOptions {
666    pub fn new() -> Self {
667        Self::default()
668    }
669
670    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
671        self.task_queue = Some(task_queue.into());
672        self
673    }
674
675    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
676        self.retry_policy = Some(policy);
677        self
678    }
679
680    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
681        self.start_to_close_timeout = Some(timeout);
682        self
683    }
684
685    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
686        self.schedule_to_start_timeout = Some(timeout);
687        self
688    }
689
690    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
691        self.schedule_to_close_timeout = Some(timeout);
692        self
693    }
694
695    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
696        self.heartbeat_timeout = Some(timeout);
697        self
698    }
699
700    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
701        if self
702            .task_queue
703            .as_deref()
704            .is_some_and(|queue| queue.trim().is_empty())
705        {
706            return Err(ActivityOptionsError::new(
707                ActivityOptionsErrorKind::EmptyTaskQueue,
708                Some("task_queue"),
709                "task_queue must not be empty",
710            ));
711        }
712
713        for (field, value) in [
714            ("start_to_close_timeout", self.start_to_close_timeout),
715            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
716            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
717            ("heartbeat_timeout", self.heartbeat_timeout),
718        ] {
719            if value.is_some_and(|value| value.is_zero()) {
720                return Err(ActivityOptionsError::new(
721                    ActivityOptionsErrorKind::TimeoutNotPositive,
722                    Some(field),
723                    format!("{field} must be positive"),
724                ));
725            }
726        }
727
728        validate_timeout_order(
729            "heartbeat_timeout",
730            self.heartbeat_timeout,
731            "start_to_close_timeout",
732            self.start_to_close_timeout,
733        )?;
734        validate_timeout_order(
735            "start_to_close_timeout",
736            self.start_to_close_timeout,
737            "schedule_to_close_timeout",
738            self.schedule_to_close_timeout,
739        )?;
740        validate_timeout_order(
741            "schedule_to_start_timeout",
742            self.schedule_to_start_timeout,
743            "schedule_to_close_timeout",
744            self.schedule_to_close_timeout,
745        )?;
746
747        Ok(ValidatedActivityOptions {
748            task_queue: self.task_queue.clone(),
749            retry_policy: self
750                .retry_policy
751                .as_ref()
752                .map(validate_activity_retry_policy)
753                .transpose()?,
754            start_to_close_timeout: timeout_seconds(
755                "start_to_close_timeout",
756                self.start_to_close_timeout,
757            )?,
758            schedule_to_start_timeout: timeout_seconds(
759                "schedule_to_start_timeout",
760                self.schedule_to_start_timeout,
761            )?,
762            schedule_to_close_timeout: timeout_seconds(
763                "schedule_to_close_timeout",
764                self.schedule_to_close_timeout,
765            )?,
766            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
767        })
768    }
769}
770
771#[derive(Clone, Debug)]
772struct ValidatedActivityOptions {
773    task_queue: Option<String>,
774    retry_policy: Option<Value>,
775    start_to_close_timeout: Option<u64>,
776    schedule_to_start_timeout: Option<u64>,
777    schedule_to_close_timeout: Option<u64>,
778    heartbeat_timeout: Option<u64>,
779}
780
781fn validate_timeout_order(
782    smaller_name: &'static str,
783    smaller: Option<Duration>,
784    larger_name: &'static str,
785    larger: Option<Duration>,
786) -> std::result::Result<(), ActivityOptionsError> {
787    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
788        return Err(ActivityOptionsError::new(
789            ActivityOptionsErrorKind::TimeoutOrder,
790            Some(smaller_name),
791            format!("{smaller_name} must be <= {larger_name}"),
792        ));
793    }
794    Ok(())
795}
796
797fn timeout_seconds(
798    field: &'static str,
799    value: Option<Duration>,
800) -> std::result::Result<Option<u64>, ActivityOptionsError> {
801    value
802        .map(|value| {
803            activity_protocol_seconds(value).ok_or_else(|| {
804                ActivityOptionsError::new(
805                    ActivityOptionsErrorKind::TimeoutOverflow,
806                    Some(field),
807                    format!("{field} is too large for the worker protocol"),
808                )
809            })
810        })
811        .transpose()
812}
813
814fn duration_seconds_ceil(value: Duration) -> Option<u64> {
815    value
816        .as_secs()
817        .checked_add(u64::from(value.subsec_nanos() > 0))
818}
819
820fn activity_protocol_seconds(value: Duration) -> Option<u64> {
821    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
822}
823
824fn validate_activity_retry_policy(
825    policy: &ActivityRetryPolicy,
826) -> std::result::Result<Value, ActivityOptionsError> {
827    if policy.max_attempts.is_none()
828        && policy.backoff.is_none()
829        && policy.non_retryable_error_types.is_empty()
830    {
831        return Err(ActivityOptionsError::new(
832            ActivityOptionsErrorKind::EmptyRetryPolicy,
833            Some("retry_policy"),
834            "retry_policy must configure at least one field",
835        ));
836    }
837    if policy.max_attempts == Some(0) {
838        return Err(ActivityOptionsError::new(
839            ActivityOptionsErrorKind::InvalidMaxAttempts,
840            Some("retry_policy.max_attempts"),
841            "max_attempts must be >= 1",
842        ));
843    }
844    if policy
845        .non_retryable_error_types
846        .iter()
847        .any(|error_type| error_type.trim().is_empty())
848    {
849        return Err(ActivityOptionsError::new(
850            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
851            Some("retry_policy.non_retryable_error_types"),
852            "non_retryable_error_types must not contain empty values",
853        ));
854    }
855
856    let backoff_seconds = match &policy.backoff {
857        None => None,
858        Some(backoff) => {
859            let max_attempts = policy.max_attempts.ok_or_else(|| {
860                ActivityOptionsError::new(
861                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
862                    Some("retry_policy.backoff"),
863                    "backoff requires max_attempts",
864                )
865            })?;
866            let retry_count = max_attempts.saturating_sub(1) as usize;
867            let intervals = match backoff {
868                ActivityBackoff::Explicit(intervals) => {
869                    if intervals.len() > retry_count {
870                        return Err(ActivityOptionsError::new(
871                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
872                            Some("retry_policy.backoff"),
873                            "backoff interval count must not exceed max_attempts - 1",
874                        ));
875                    }
876                    intervals.clone()
877                }
878                ActivityBackoff::Exponential {
879                    initial_interval,
880                    coefficient,
881                    maximum_interval,
882                } => {
883                    if *coefficient < 1 {
884                        return Err(ActivityOptionsError::new(
885                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
886                            Some("retry_policy.backoff.coefficient"),
887                            "backoff coefficient must be >= 1",
888                        ));
889                    }
890                    if retry_count > 10_000 {
891                        return Err(ActivityOptionsError::new(
892                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
893                            Some("retry_policy.max_attempts"),
894                            "generated backoff supports at most 10000 retry intervals",
895                        ));
896                    }
897                    let mut current = *initial_interval;
898                    let mut intervals = Vec::with_capacity(retry_count);
899                    for _ in 0..retry_count {
900                        let interval = maximum_interval
901                            .map(|maximum| current.min(maximum))
902                            .unwrap_or(current);
903                        intervals.push(interval);
904                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
905                            break;
906                        }
907                        current = current.checked_mul(*coefficient).ok_or_else(|| {
908                            ActivityOptionsError::new(
909                                ActivityOptionsErrorKind::BackoffOverflow,
910                                Some("retry_policy.backoff"),
911                                "generated backoff interval overflowed",
912                            )
913                        })?;
914                    }
915                    intervals
916                }
917            };
918            Some(
919                intervals
920                    .into_iter()
921                    .map(|interval| {
922                        activity_protocol_seconds(interval).ok_or_else(|| {
923                            ActivityOptionsError::new(
924                                ActivityOptionsErrorKind::BackoffOverflow,
925                                Some("retry_policy.backoff"),
926                                "backoff interval is too large for the worker protocol",
927                            )
928                        })
929                    })
930                    .collect::<std::result::Result<Vec<_>, _>>()?,
931            )
932        }
933    };
934
935    let mut encoded = serde_json::Map::new();
936    if let Some(max_attempts) = policy.max_attempts {
937        encoded.insert("max_attempts".to_string(), json!(max_attempts));
938    }
939    if let Some(backoff_seconds) = backoff_seconds {
940        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
941    }
942    if !policy.non_retryable_error_types.is_empty() {
943        let mut canonical_error_types = Vec::new();
944        for error_type in policy
945            .non_retryable_error_types
946            .iter()
947            .map(|error_type| error_type.trim())
948        {
949            if !canonical_error_types.contains(&error_type) {
950                canonical_error_types.push(error_type);
951            }
952        }
953        encoded.insert(
954            "non_retryable_error_types".to_string(),
955            json!(canonical_error_types),
956        );
957    }
958    Ok(Value::Object(encoded))
959}
960
961/// A stable, machine-readable failure raised when workflow code no longer
962/// reconstructs the durable command stream recorded in history.
963#[derive(Clone, Debug, Error)]
964#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
965pub struct ReplayFailure {
966    pub reason: String,
967    pub sequence: Option<u64>,
968    pub expected: Option<String>,
969    pub actual: Option<String>,
970    pub message: String,
971}
972
973impl ReplayFailure {
974    fn new(
975        reason: impl Into<String>,
976        sequence: Option<u64>,
977        expected: Option<String>,
978        actual: Option<String>,
979        message: impl Into<String>,
980    ) -> Self {
981        Self {
982            reason: reason.into(),
983            sequence,
984            expected,
985            actual,
986            message: message.into(),
987        }
988    }
989}
990
991/// A stable, machine-readable workflow query or query-task settlement failure.
992#[derive(Clone, Debug, Error)]
993#[error("query failed ({reason}, HTTP {status}): {message}")]
994pub struct QueryFailure {
995    pub status: u16,
996    pub reason: String,
997    pub message: String,
998    pub body: Value,
999}
1000
1001/// A stable failure returned when a server rejects an SDK protocol version.
1002#[derive(Clone, Debug, Error)]
1003#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1004pub struct ProtocolFailure {
1005    pub status: u16,
1006    pub reason: String,
1007    pub message: String,
1008    pub supported_version: Option<String>,
1009    pub requested_version: Option<String>,
1010    pub body: Value,
1011}
1012
1013#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1014pub struct PayloadEnvelope {
1015    pub codec: String,
1016    pub blob: String,
1017}
1018
1019impl PayloadEnvelope {
1020    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1021        encode_payload(value, DEFAULT_CODEC)
1022    }
1023
1024    /// Encode an explicit typed value, including the bytes branch that JSON
1025    /// serialization cannot represent.
1026    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1027        encode_avro_value(value)
1028    }
1029}
1030
1031/// Native adapter for the fixed language-neutral Avro Value schema.
1032#[derive(Clone, Debug, PartialEq)]
1033pub enum AvroValue {
1034    Null,
1035    Boolean(bool),
1036    Long(i64),
1037    Double(f64),
1038    Bytes(Vec<u8>),
1039    String(String),
1040    Array(Vec<AvroValue>),
1041    Map(BTreeMap<String, AvroValue>),
1042}
1043
1044impl AvroValue {
1045    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1046        Self::from_serde_value(
1047            serde_value::to_value(value).map_err(|error| {
1048                Error::Codec(format!("could not adapt value for Avro: {error}"))
1049            })?,
1050        )
1051    }
1052
1053    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1054        use serde_value::Value as SerdeValue;
1055
1056        match value {
1057            SerdeValue::Unit => Ok(Self::Null),
1058            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1059            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1060            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1061            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1062            SerdeValue::I64(value) => Ok(Self::Long(value)),
1063            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1064            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1065            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1066            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1067                Error::Codec(
1068                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1069                        .to_string(),
1070                )
1071            }),
1072            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1073            SerdeValue::F64(value) => Self::finite_double(value),
1074            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1075            SerdeValue::String(value) => Ok(Self::String(value)),
1076            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1077            SerdeValue::Option(None) => Ok(Self::Null),
1078            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1079                Self::from_serde_value(*value)
1080            }
1081            SerdeValue::Seq(values) => values
1082                .into_iter()
1083                .map(Self::from_serde_value)
1084                .collect::<Result<Vec<_>>>()
1085                .map(Self::Array),
1086            SerdeValue::Map(values) => values
1087                .into_iter()
1088                .map(|(key, value)| {
1089                    let SerdeValue::String(key) = key else {
1090                        return Err(Error::Codec(
1091                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1092                        ));
1093                    };
1094
1095                    Ok((key, Self::from_serde_value(value)?))
1096                })
1097                .collect::<Result<BTreeMap<_, _>>>()
1098                .map(Self::Map),
1099        }
1100    }
1101
1102    fn finite_double(value: f64) -> Result<Self> {
1103        if !value.is_finite() {
1104            return Err(Error::Codec(
1105                "non_finite_float: Avro Value doubles must be finite".to_string(),
1106            ));
1107        }
1108
1109        Ok(Self::Double(value))
1110    }
1111
1112    fn into_json(self) -> Result<Value> {
1113        match self {
1114            Self::Null => Ok(Value::Null),
1115            Self::Boolean(value) => Ok(Value::Bool(value)),
1116            Self::Long(value) => Ok(Value::Number(value.into())),
1117            Self::Double(value) => serde_json::Number::from_f64(value)
1118                .map(Value::Number)
1119                .ok_or_else(|| {
1120                    Error::Codec(
1121                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1122                    )
1123                }),
1124            Self::Bytes(value) => Ok(json!({
1125                "$type": "bytes",
1126                "base64": BASE64.encode(value),
1127            })),
1128            Self::String(value) => Ok(Value::String(value)),
1129            Self::Array(values) => values
1130                .into_iter()
1131                .map(Self::into_json)
1132                .collect::<Result<Vec<_>>>()
1133                .map(Value::Array),
1134            Self::Map(values) => values
1135                .into_iter()
1136                .map(|(key, value)| Ok((key, value.into_json()?)))
1137                .collect::<Result<serde_json::Map<_, _>>>()
1138                .map(Value::Object),
1139        }
1140    }
1141
1142    fn into_serde_value(self) -> serde_value::Value {
1143        use serde_value::Value as SerdeValue;
1144
1145        match self {
1146            Self::Null => SerdeValue::Unit,
1147            Self::Boolean(value) => SerdeValue::Bool(value),
1148            Self::Long(value) => SerdeValue::I64(value),
1149            Self::Double(value) => SerdeValue::F64(value),
1150            Self::Bytes(value) => SerdeValue::Bytes(value),
1151            Self::String(value) => SerdeValue::String(value),
1152            Self::Array(values) => {
1153                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1154            }
1155            Self::Map(values) => SerdeValue::Map(
1156                values
1157                    .into_iter()
1158                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1159                    .collect(),
1160            ),
1161        }
1162    }
1163
1164    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1165        self.into_serde_value().deserialize_into().map_err(|error| {
1166            Error::Codec(format!(
1167                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1168            ))
1169        })
1170    }
1171}
1172
1173impl Serialize for AvroValue {
1174    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1175    where
1176        S: Serializer,
1177    {
1178        match self {
1179            Self::Null => serializer.serialize_unit(),
1180            Self::Boolean(value) => serializer.serialize_bool(*value),
1181            Self::Long(value) => serializer.serialize_i64(*value),
1182            Self::Double(value) => serializer.serialize_f64(*value),
1183            Self::Bytes(value) => serializer.serialize_bytes(value),
1184            Self::String(value) => serializer.serialize_str(value),
1185            Self::Array(values) => {
1186                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1187                for value in values {
1188                    sequence.serialize_element(value)?;
1189                }
1190                sequence.end()
1191            }
1192            Self::Map(values) => {
1193                let mut map = serializer.serialize_map(Some(values.len()))?;
1194                for (key, value) in values {
1195                    map.serialize_entry(key, value)?;
1196                }
1197                map.end()
1198            }
1199        }
1200    }
1201}
1202
1203pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1204    let datum = avro_value_to_datum(value)?;
1205    let datum = to_avro_datum(avro_value_schema()?, datum)
1206        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1207    let mut bytes = Vec::with_capacity(datum.len() + 10);
1208    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1209    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1210    bytes.extend_from_slice(&datum);
1211    Ok(PayloadEnvelope {
1212        codec: DEFAULT_CODEC.to_string(),
1213        blob: BASE64.encode(bytes),
1214    })
1215}
1216
1217pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1218    if envelope.codec != DEFAULT_CODEC {
1219        return Err(unsupported_payload_codec(&envelope.codec));
1220    }
1221    decode_avro_value_blob(&envelope.blob)
1222}
1223
1224pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1225    let blob = match codec {
1226        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1227        other => return Err(unsupported_payload_codec(other)),
1228    };
1229
1230    Ok(PayloadEnvelope {
1231        codec: codec.to_string(),
1232        blob,
1233    })
1234}
1235
1236pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1237    match envelope.codec.as_str() {
1238        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1239        other => Err(unsupported_payload_codec(other)),
1240    }
1241}
1242
1243#[cfg(test)]
1244fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1245    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1246}
1247
1248fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1249    validate_payload_codec(fallback_codec)?;
1250
1251    if value.is_null() {
1252        return Ok(Value::Null);
1253    }
1254
1255    if let Some((codec, blob)) = payload_envelope_parts(value)? {
1256        return decode_blob(blob, codec);
1257    }
1258
1259    if let Some(blob) = value.as_str() {
1260        return decode_blob(blob, fallback_codec);
1261    }
1262
1263    Err(untagged_payload_value())
1264}
1265
1266fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1267    let envelope = match codec {
1268        DEFAULT_CODEC => encode_avro_value(value)?,
1269        other => return Err(unsupported_payload_codec(other)),
1270    };
1271    Ok(serde_json::to_value(envelope)?)
1272}
1273
1274fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1275    validate_payload_codec(fallback_codec)?;
1276
1277    if value.is_null() {
1278        return Ok(AvroValue::Null);
1279    }
1280
1281    if let Some((codec, blob)) = payload_envelope_parts(value)? {
1282        validate_payload_codec(codec)?;
1283        return decode_avro_value_blob(blob);
1284    }
1285
1286    if let Some(blob) = value.as_str() {
1287        return match fallback_codec {
1288            DEFAULT_CODEC => decode_avro_value_blob(blob),
1289            other => Err(unsupported_payload_codec(other)),
1290        };
1291    }
1292
1293    Err(untagged_payload_value())
1294}
1295
1296fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1297    match value {
1298        AvroValue::Null => AvroValue::Array(Vec::new()),
1299        AvroValue::Array(_) => value,
1300        other => AvroValue::Array(vec![other]),
1301    }
1302}
1303
1304fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1305    match codec {
1306        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1307        other => Err(unsupported_payload_codec(other)),
1308    }
1309}
1310
1311fn validate_payload_codec(codec: &str) -> Result<()> {
1312    match codec {
1313        DEFAULT_CODEC => Ok(()),
1314        other => Err(unsupported_payload_codec(other)),
1315    }
1316}
1317
1318fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
1319    let Some(object) = value.as_object() else {
1320        return Ok(None);
1321    };
1322    if !object.contains_key("codec") && !object.contains_key("blob") {
1323        return Ok(None);
1324    }
1325
1326    let codec = object
1327        .get("codec")
1328        .and_then(Value::as_str)
1329        .ok_or_else(invalid_payload_envelope)?;
1330    validate_payload_codec(codec)?;
1331    let blob = object
1332        .get("blob")
1333        .and_then(Value::as_str)
1334        .ok_or_else(invalid_payload_envelope)?;
1335    Ok(Some((codec, blob)))
1336}
1337
1338fn invalid_payload_envelope() -> Error {
1339    Error::Codec(
1340        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
1341            .to_string(),
1342    )
1343}
1344
1345fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
1346    for command in commands {
1347        let Some(command) = command.as_object() else {
1348            continue;
1349        };
1350        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
1351            continue;
1352        };
1353        let Some(payload_field) = workflow_command_payload_field(command_type) else {
1354            continue;
1355        };
1356
1357        if let Some(codec) = command.get("payload_codec") {
1358            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
1359            validate_payload_codec(codec)?;
1360        }
1361
1362        let payload = command
1363            .get(payload_field)
1364            .ok_or_else(invalid_payload_envelope)?;
1365        validate_outbound_payload_envelope(payload)?;
1366    }
1367    Ok(())
1368}
1369
1370fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
1371    match command_type {
1372        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
1373        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
1374        "start_service_operation" => Some("request_payload"),
1375        _ => None,
1376    }
1377}
1378
1379fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
1380    let Some((codec, blob)) = payload_envelope_parts(value)? else {
1381        return Err(untagged_payload_value());
1382    };
1383    validate_payload_codec(codec)?;
1384    decode_avro_value_blob(blob)?;
1385    Ok(())
1386}
1387
1388fn unsupported_payload_codec(codec: &str) -> Error {
1389    Error::Codec(format!(
1390        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1391    ))
1392}
1393
1394fn untagged_payload_value() -> Error {
1395    Error::Codec(
1396        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1397            .to_string(),
1398    )
1399}
1400
1401fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1402    let bytes = BASE64.decode(blob).map_err(|err| {
1403        Error::Codec(format!(
1404            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1405        ))
1406    })?;
1407
1408    if serde_json::from_slice::<Value>(&bytes).is_ok() {
1409        return Err(unsupported_payload_codec("json"));
1410    }
1411
1412    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1413        return Err(Error::Codec(
1414            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1415        ));
1416    }
1417
1418    let fingerprint: [u8; 8] = bytes[2..10]
1419        .try_into()
1420        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1421    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1422        return Err(Error::Codec(format!(
1423            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1424            fingerprint
1425                .iter()
1426                .map(|byte| format!("{byte:02x}"))
1427                .collect::<String>()
1428        )));
1429    }
1430
1431    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1432    // The current fingerprint selects the current immutable schema, so reader
1433    // resolution would only re-walk the same recursive union. Future retained
1434    // writer fingerprints supply a distinct reader schema in this branch.
1435    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1436    if datum_reader.truncated {
1437        return Err(Error::Codec(
1438            "invalid_payload_framing: truncated Avro Value datum".to_string(),
1439        ));
1440    }
1441    let datum = datum.map_err(|err| {
1442        Error::Codec(format!(
1443            "invalid_payload_framing: malformed Avro Value datum: {err}"
1444        ))
1445    })?;
1446    if datum_reader.remaining() != 0 {
1447        return Err(Error::Codec(format!(
1448            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1449            datum_reader.remaining()
1450        )));
1451    }
1452    avro_value_from_datum(datum)
1453}
1454
1455struct StrictAvroDatumReader<'a> {
1456    bytes: &'a [u8],
1457    offset: usize,
1458    truncated: bool,
1459}
1460
1461impl<'a> StrictAvroDatumReader<'a> {
1462    fn new(bytes: &'a [u8]) -> Self {
1463        Self {
1464            bytes,
1465            offset: 0,
1466            truncated: false,
1467        }
1468    }
1469
1470    fn remaining(&self) -> usize {
1471        self.bytes.len() - self.offset
1472    }
1473}
1474
1475impl Read for StrictAvroDatumReader<'_> {
1476    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1477        let count = buffer.len().min(self.remaining());
1478        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1479        self.offset += count;
1480        if count < buffer.len() {
1481            self.truncated = true;
1482        }
1483
1484        Ok(count)
1485    }
1486}
1487
1488fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1489    let branch = match value {
1490        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1491        AvroValue::Boolean(value) => AvroDatum::Union(
1492            1,
1493            Box::new(AvroDatum::Record(vec![(
1494                "boolean".to_string(),
1495                AvroDatum::Boolean(*value),
1496            )])),
1497        ),
1498        AvroValue::Long(value) => AvroDatum::Union(
1499            2,
1500            Box::new(AvroDatum::Record(vec![(
1501                "long".to_string(),
1502                AvroDatum::Long(*value),
1503            )])),
1504        ),
1505        AvroValue::Double(value) => {
1506            if !value.is_finite() {
1507                return Err(Error::Codec(
1508                    "non_finite_float: Avro Value doubles must be finite".to_string(),
1509                ));
1510            }
1511            AvroDatum::Union(
1512                3,
1513                Box::new(AvroDatum::Record(vec![(
1514                    "double".to_string(),
1515                    AvroDatum::Double(*value),
1516                )])),
1517            )
1518        }
1519        AvroValue::Bytes(value) => AvroDatum::Union(
1520            4,
1521            Box::new(AvroDatum::Record(vec![(
1522                "bytes".to_string(),
1523                AvroDatum::Bytes(value.clone()),
1524            )])),
1525        ),
1526        AvroValue::String(value) => AvroDatum::Union(
1527            5,
1528            Box::new(AvroDatum::Record(vec![(
1529                "string".to_string(),
1530                AvroDatum::String(value.clone()),
1531            )])),
1532        ),
1533        AvroValue::Array(values) => AvroDatum::Union(
1534            6,
1535            Box::new(AvroDatum::Record(vec![(
1536                "items".to_string(),
1537                AvroDatum::Array(
1538                    values
1539                        .iter()
1540                        .map(avro_value_to_datum)
1541                        .collect::<Result<Vec<_>>>()?,
1542                ),
1543            )])),
1544        ),
1545        AvroValue::Map(values) => AvroDatum::Union(
1546            7,
1547            Box::new(AvroDatum::Record(vec![(
1548                "entries".to_string(),
1549                AvroDatum::Map(
1550                    values
1551                        .iter()
1552                        .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1553                        .collect::<Result<HashMap<_, _>>>()?,
1554                ),
1555            )])),
1556        ),
1557    };
1558    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1559}
1560
1561fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1562    let AvroDatum::Record(mut outer) = datum else {
1563        return Err(Error::Codec(
1564            "invalid_payload_framing: datum is not a Value record".to_string(),
1565        ));
1566    };
1567    let (_, branch) = outer
1568        .pop()
1569        .filter(|(name, _)| name == "value")
1570        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1571    let AvroDatum::Union(_, branch) = branch else {
1572        return Err(Error::Codec(
1573            "invalid_payload_framing: invalid Value union".to_string(),
1574        ));
1575    };
1576    match *branch {
1577        AvroDatum::Null => Ok(AvroValue::Null),
1578        AvroDatum::Record(mut fields) => {
1579            let (name, value) = fields.pop().ok_or_else(|| {
1580                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1581            })?;
1582            match (name.as_str(), value) {
1583                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1584                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1585                ("double", AvroDatum::Double(value)) if value.is_finite() => {
1586                    Ok(AvroValue::Double(value))
1587                }
1588                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1589                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1590                ("items", AvroDatum::Array(values)) => values
1591                    .into_iter()
1592                    .map(avro_value_from_datum)
1593                    .collect::<Result<Vec<_>>>()
1594                    .map(AvroValue::Array),
1595                ("entries", AvroDatum::Map(values)) => values
1596                    .into_iter()
1597                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1598                    .collect::<Result<BTreeMap<_, _>>>()
1599                    .map(AvroValue::Map),
1600                _ => Err(Error::Codec(
1601                    "invalid_payload_framing: unknown Value branch".to_string(),
1602                )),
1603            }
1604        }
1605        _ => Err(Error::Codec(
1606            "invalid_payload_framing: invalid Value branch".to_string(),
1607        )),
1608    }
1609}
1610
1611fn avro_value_schema() -> Result<&'static Schema> {
1612    match AVRO_VALUE_SCHEMA.get_or_init(|| {
1613        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1614            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1615    }) {
1616        Ok(schema) => Ok(schema),
1617        Err(message) => Err(Error::Codec(message.clone())),
1618    }
1619}
1620
1621#[derive(Clone, Debug)]
1622pub struct Client {
1623    http: reqwest::Client,
1624    base_url: String,
1625    token: Option<String>,
1626    control_token: Option<String>,
1627    worker_token: Option<String>,
1628    namespace: String,
1629}
1630
1631impl Client {
1632    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1633        Self::builder(base_url).build()
1634    }
1635
1636    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1637        ClientBuilder {
1638            base_url: base_url.into(),
1639            token: None,
1640            control_token: None,
1641            worker_token: None,
1642            namespace: "default".to_string(),
1643            timeout: Duration::from_secs(60),
1644        }
1645    }
1646
1647    pub async fn health(&self) -> Result<Value> {
1648        self.request_json(
1649            reqwest::Method::GET,
1650            "/health",
1651            RequestProtocol::ControlPlane,
1652            Option::<&Value>::None,
1653        )
1654        .await
1655    }
1656
1657    pub async fn cluster_info(&self) -> Result<Value> {
1658        self.request_json(
1659            reqwest::Method::GET,
1660            "/cluster/info",
1661            RequestProtocol::ControlPlane,
1662            Option::<&Value>::None,
1663        )
1664        .await
1665    }
1666
1667    pub async fn start_workflow<T: Serialize>(
1668        &self,
1669        workflow_type: &str,
1670        task_queue: &str,
1671        workflow_id: &str,
1672        input: T,
1673    ) -> Result<WorkflowHandle> {
1674        self.start_workflow_with_options(
1675            workflow_type,
1676            task_queue,
1677            workflow_id,
1678            WorkflowStartOptions::default(),
1679            input,
1680        )
1681        .await
1682    }
1683
1684    /// Start a workflow with explicit server-enforced execution and run
1685    /// deadlines.
1686    pub async fn start_workflow_with_options<T: Serialize>(
1687        &self,
1688        workflow_type: &str,
1689        task_queue: &str,
1690        workflow_id: &str,
1691        options: WorkflowStartOptions,
1692        input: T,
1693    ) -> Result<WorkflowHandle> {
1694        options.validate()?;
1695        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1696        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1697        let body = json!({
1698            "workflow_id": workflow_id,
1699            "workflow_type": workflow_type,
1700            "task_queue": task_queue,
1701            "input": input_envelope,
1702            "execution_timeout_seconds": options.execution_timeout_seconds,
1703            "run_timeout_seconds": options.run_timeout_seconds
1704        });
1705
1706        let data: Value = self
1707            .request_json(
1708                reqwest::Method::POST,
1709                "/workflows",
1710                RequestProtocol::ControlPlane,
1711                Some(&body),
1712            )
1713            .await?;
1714
1715        Ok(WorkflowHandle {
1716            client: self.clone(),
1717            workflow_id: data
1718                .get("workflow_id")
1719                .and_then(Value::as_str)
1720                .unwrap_or(workflow_id)
1721                .to_string(),
1722            run_id: data
1723                .get("run_id")
1724                .and_then(Value::as_str)
1725                .map(str::to_string),
1726            workflow_type: data
1727                .get("workflow_type")
1728                .and_then(Value::as_str)
1729                .unwrap_or(workflow_type)
1730                .to_string(),
1731        })
1732    }
1733
1734    pub async fn signal_workflow<T: Serialize>(
1735        &self,
1736        workflow_id: &str,
1737        signal_name: &str,
1738        input: T,
1739    ) -> Result<Value> {
1740        self.signal_workflow_target(workflow_id, None, signal_name, input)
1741            .await
1742    }
1743
1744    /// Signal only if `run_id` is still the current run for this instance.
1745    pub async fn signal_workflow_run<T: Serialize>(
1746        &self,
1747        workflow_id: &str,
1748        run_id: &str,
1749        signal_name: &str,
1750        input: T,
1751    ) -> Result<Value> {
1752        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1753            .await
1754    }
1755
1756    async fn signal_workflow_target<T: Serialize>(
1757        &self,
1758        workflow_id: &str,
1759        run_id: Option<&str>,
1760        signal_name: &str,
1761        input: T,
1762    ) -> Result<Value> {
1763        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1764        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1765        let body = json!({
1766            "input": input_envelope
1767        });
1768        let path = match run_id {
1769            Some(run_id) => {
1770                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1771            }
1772            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1773        };
1774        self.request_json(
1775            reqwest::Method::POST,
1776            &path,
1777            RequestProtocol::ControlPlane,
1778            Some(&body),
1779        )
1780        .await
1781    }
1782
1783    /// Request cooperative cancellation of the current run for an instance.
1784    pub async fn cancel_workflow(
1785        &self,
1786        workflow_id: &str,
1787        options: WorkflowCommandOptions,
1788    ) -> Result<WorkflowCommandResult> {
1789        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1790            .await
1791    }
1792
1793    /// Request cooperative cancellation only if `run_id` is still current.
1794    pub async fn cancel_workflow_run(
1795        &self,
1796        workflow_id: &str,
1797        run_id: &str,
1798        options: WorkflowCommandOptions,
1799    ) -> Result<WorkflowCommandResult> {
1800        self.workflow_command(
1801            workflow_id,
1802            Some(run_id),
1803            WorkflowCommandKind::Cancel,
1804            options,
1805        )
1806        .await
1807    }
1808
1809    /// Forcefully terminate the current run for an instance.
1810    pub async fn terminate_workflow(
1811        &self,
1812        workflow_id: &str,
1813        options: WorkflowCommandOptions,
1814    ) -> Result<WorkflowCommandResult> {
1815        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1816            .await
1817    }
1818
1819    /// Forcefully terminate only if `run_id` is still current.
1820    pub async fn terminate_workflow_run(
1821        &self,
1822        workflow_id: &str,
1823        run_id: &str,
1824        options: WorkflowCommandOptions,
1825    ) -> Result<WorkflowCommandResult> {
1826        self.workflow_command(
1827            workflow_id,
1828            Some(run_id),
1829            WorkflowCommandKind::Terminate,
1830            options,
1831        )
1832        .await
1833    }
1834
1835    async fn workflow_command(
1836        &self,
1837        workflow_id: &str,
1838        run_id: Option<&str>,
1839        command: WorkflowCommandKind,
1840        options: WorkflowCommandOptions,
1841    ) -> Result<WorkflowCommandResult> {
1842        let path = match run_id {
1843            Some(run_id) => format!(
1844                "/workflows/{workflow_id}/runs/{run_id}/{}",
1845                command.as_str()
1846            ),
1847            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1848        };
1849        let data = match self
1850            .request_json(
1851                reqwest::Method::POST,
1852                &path,
1853                RequestProtocol::ControlPlane,
1854                Some(&options),
1855            )
1856            .await
1857        {
1858            Ok(data) => data,
1859            Err(Error::Http { status, body }) => {
1860                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1861                    command,
1862                    status,
1863                    body,
1864                    workflow_id,
1865                    run_id,
1866                )));
1867            }
1868            Err(error) => return Err(error),
1869        };
1870
1871        Ok(workflow_command_result(command, data, workflow_id, run_id))
1872    }
1873
1874    /// Execute a named, read-only query against a running or completed workflow.
1875    ///
1876    /// Arguments and results use the platform payload envelope. Server and
1877    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1878    /// reason, HTTP status, and original response body.
1879    pub async fn query_workflow<T: Serialize>(
1880        &self,
1881        workflow_id: &str,
1882        query_name: &str,
1883        input: T,
1884    ) -> Result<Value> {
1885        self.query_workflow_target(workflow_id, None, query_name, input)
1886            .await
1887    }
1888
1889    /// Query only if `run_id` is still current, preventing accidental retargeting.
1890    pub async fn query_workflow_run<T: Serialize>(
1891        &self,
1892        workflow_id: &str,
1893        run_id: &str,
1894        query_name: &str,
1895        input: T,
1896    ) -> Result<Value> {
1897        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1898            .await
1899    }
1900
1901    /// Query a workflow and return the lossless fixed Avro Value result.
1902    pub async fn query_workflow_avro_value<T: Serialize>(
1903        &self,
1904        workflow_id: &str,
1905        query_name: &str,
1906        input: T,
1907    ) -> Result<AvroValue> {
1908        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1909            .await
1910    }
1911
1912    /// Query a selected run and return the lossless fixed Avro Value result.
1913    pub async fn query_workflow_run_avro_value<T: Serialize>(
1914        &self,
1915        workflow_id: &str,
1916        run_id: &str,
1917        query_name: &str,
1918        input: T,
1919    ) -> Result<AvroValue> {
1920        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1921            .await
1922    }
1923
1924    async fn query_workflow_avro_value_target<T: Serialize>(
1925        &self,
1926        workflow_id: &str,
1927        run_id: Option<&str>,
1928        query_name: &str,
1929        input: T,
1930    ) -> Result<AvroValue> {
1931        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1932        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1933        let path = match run_id {
1934            Some(run_id) => {
1935                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1936            }
1937            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1938        };
1939        let response: Value = match self
1940            .request_json(
1941                reqwest::Method::POST,
1942                &path,
1943                RequestProtocol::ControlPlane,
1944                Some(&body),
1945            )
1946            .await
1947        {
1948            Ok(response) => response,
1949            Err(Error::Http { status, body }) => {
1950                return Err(Error::QueryFailed(query_failure(status, body)));
1951            }
1952            Err(error) => return Err(error),
1953        };
1954
1955        let envelope = response
1956            .get("result_envelope")
1957            .filter(|envelope| !envelope.is_null())
1958            .ok_or_else(|| {
1959                Error::Codec(
1960                    "missing_payload_envelope: typed query result requires result_envelope"
1961                        .to_string(),
1962                )
1963            })?;
1964        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1965    }
1966
1967    async fn query_workflow_target<T: Serialize>(
1968        &self,
1969        workflow_id: &str,
1970        run_id: Option<&str>,
1971        query_name: &str,
1972        input: T,
1973    ) -> Result<Value> {
1974        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1975        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1976        let body = json!({
1977            "input": input_envelope
1978        });
1979        let path = match run_id {
1980            Some(run_id) => {
1981                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1982            }
1983            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1984        };
1985        let response: Value = match self
1986            .request_json(
1987                reqwest::Method::POST,
1988                &path,
1989                RequestProtocol::ControlPlane,
1990                Some(&body),
1991            )
1992            .await
1993        {
1994            Ok(response) => response,
1995            Err(Error::Http { status, body }) => {
1996                return Err(Error::QueryFailed(query_failure(status, body)));
1997            }
1998            Err(error) => return Err(error),
1999        };
2000
2001        if let Some(envelope) = response
2002            .get("result_envelope")
2003            .filter(|envelope| !envelope.is_null())
2004        {
2005            return decode_wire_value(envelope, DEFAULT_CODEC);
2006        }
2007
2008        Ok(response.get("result").cloned().unwrap_or(Value::Null))
2009    }
2010
2011    /// Send a synchronous update using fixed Avro Value arguments.
2012    pub async fn update_workflow<T: Serialize>(
2013        &self,
2014        workflow_id: &str,
2015        update_name: &str,
2016        input: T,
2017        request_id: Option<&str>,
2018    ) -> Result<Value> {
2019        let response = self
2020            .update_workflow_response(workflow_id, update_name, input, request_id)
2021            .await?;
2022        if let Some(envelope) = response
2023            .get("result_envelope")
2024            .filter(|envelope| !envelope.is_null())
2025        {
2026            return decode_wire_value(envelope, DEFAULT_CODEC);
2027        }
2028        Ok(response.get("result").cloned().unwrap_or(response))
2029    }
2030
2031    /// Send a synchronous update and retain a bytes-capable Avro result.
2032    pub async fn update_workflow_avro_value<T: Serialize>(
2033        &self,
2034        workflow_id: &str,
2035        update_name: &str,
2036        input: T,
2037        request_id: Option<&str>,
2038    ) -> Result<AvroValue> {
2039        let response = self
2040            .update_workflow_response(workflow_id, update_name, input, request_id)
2041            .await?;
2042        let envelope = response
2043            .get("result_envelope")
2044            .filter(|envelope| !envelope.is_null())
2045            .ok_or_else(|| {
2046                Error::Codec(
2047                    "missing_payload_envelope: typed update result requires result_envelope"
2048                        .to_string(),
2049                )
2050            })?;
2051        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2052    }
2053
2054    async fn update_workflow_response<T: Serialize>(
2055        &self,
2056        workflow_id: &str,
2057        update_name: &str,
2058        input: T,
2059        request_id: Option<&str>,
2060    ) -> Result<Value> {
2061        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2062        let mut body = json!({
2063            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2064            "wait_for": "completed",
2065        });
2066        if let Some(request_id) = request_id {
2067            body["request_id"] = json!(request_id);
2068        }
2069        self.request_json(
2070            reqwest::Method::POST,
2071            &format!("/workflows/{workflow_id}/update/{update_name}"),
2072            RequestProtocol::ControlPlane,
2073            Some(&body),
2074        )
2075        .await
2076    }
2077
2078    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2079        let path = format!("/workflows/{workflow_id}");
2080        let mut data: WorkflowDescription = self
2081            .request_json(
2082                reqwest::Method::GET,
2083                &path,
2084                RequestProtocol::ControlPlane,
2085                Option::<&Value>::None,
2086            )
2087            .await?;
2088        data.decode_payloads()?;
2089        Ok(data)
2090    }
2091
2092    /// Describe one selected run, including historical terminal runs.
2093    pub async fn describe_workflow_run(
2094        &self,
2095        workflow_id: &str,
2096        run_id: &str,
2097    ) -> Result<WorkflowDescription> {
2098        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2099        let mut data: WorkflowDescription = self
2100            .request_json(
2101                reqwest::Method::GET,
2102                &path,
2103                RequestProtocol::ControlPlane,
2104                Option::<&Value>::None,
2105            )
2106            .await?;
2107        data.decode_payloads()?;
2108        Ok(data)
2109    }
2110
2111    pub async fn register_worker(
2112        &self,
2113        worker_id: &str,
2114        task_queue: &str,
2115        supported_workflow_types: Vec<String>,
2116        supported_activity_types: Vec<String>,
2117        max_concurrent_workflow_tasks: usize,
2118        max_concurrent_activity_tasks: usize,
2119    ) -> Result<RegisterWorkerResponse> {
2120        self.register_worker_with_capabilities(
2121            worker_id,
2122            task_queue,
2123            supported_workflow_types,
2124            supported_activity_types,
2125            max_concurrent_workflow_tasks,
2126            max_concurrent_activity_tasks,
2127            Vec::new(),
2128        )
2129        .await
2130    }
2131
2132    /// Register a worker and explicitly advertise additive worker capabilities.
2133    pub async fn register_worker_with_capabilities(
2134        &self,
2135        worker_id: &str,
2136        task_queue: &str,
2137        supported_workflow_types: Vec<String>,
2138        supported_activity_types: Vec<String>,
2139        max_concurrent_workflow_tasks: usize,
2140        max_concurrent_activity_tasks: usize,
2141        capabilities: Vec<String>,
2142    ) -> Result<RegisterWorkerResponse> {
2143        self.register_worker_with_command_contracts(
2144            worker_id,
2145            task_queue,
2146            supported_workflow_types,
2147            supported_activity_types,
2148            max_concurrent_workflow_tasks,
2149            max_concurrent_activity_tasks,
2150            capabilities,
2151            Value::Object(serde_json::Map::new()),
2152        )
2153        .await
2154    }
2155
2156    /// Register a worker and advertise its named query and update handlers.
2157    ///
2158    /// This Rust SDK cannot execute synchronous pre-accept update validation,
2159    /// so a workflow contract with a non-empty or malformed
2160    /// `update_validators` declaration returns
2161    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
2162    #[allow(clippy::too_many_arguments)]
2163    pub async fn register_worker_with_command_contracts(
2164        &self,
2165        worker_id: &str,
2166        task_queue: &str,
2167        supported_workflow_types: Vec<String>,
2168        supported_activity_types: Vec<String>,
2169        max_concurrent_workflow_tasks: usize,
2170        max_concurrent_activity_tasks: usize,
2171        capabilities: Vec<String>,
2172        workflow_command_contracts: Value,
2173    ) -> Result<RegisterWorkerResponse> {
2174        if let Some(contracts) = workflow_command_contracts.as_object() {
2175            for (workflow_type, contract) in contracts {
2176                let Some(update_validators) = contract.get("update_validators") else {
2177                    continue;
2178                };
2179                if !update_validators
2180                    .as_array()
2181                    .is_some_and(|validators| validators.is_empty())
2182                {
2183                    return Err(Error::UnsupportedUpdateValidators {
2184                        workflow_type: workflow_type.clone(),
2185                    });
2186                }
2187            }
2188        }
2189
2190        let mut body = json!({
2191            "worker_id": worker_id,
2192            "task_queue": task_queue,
2193            "runtime": "rust",
2194            "sdk_version": SDK_VERSION,
2195            "supported_workflow_types": supported_workflow_types,
2196            "supported_activity_types": supported_activity_types,
2197            "capabilities": capabilities,
2198            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2199            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2200        });
2201        if workflow_command_contracts
2202            .as_object()
2203            .is_some_and(|contracts| !contracts.is_empty())
2204        {
2205            body["workflow_command_contracts"] = workflow_command_contracts;
2206        }
2207
2208        self.request_json(
2209            reqwest::Method::POST,
2210            "/worker/register",
2211            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2212            Some(&body),
2213        )
2214        .await
2215    }
2216
2217    /// Gracefully remove one worker's registration through the worker plane.
2218    ///
2219    /// This operation is separate from operator-facing worker management. It
2220    /// uses worker-protocol authentication and returns the server's lease
2221    /// recovery result.
2222    pub async fn deregister_worker_registration(
2223        &self,
2224        worker_id: &str,
2225    ) -> Result<WorkerDeregistrationEnvelope> {
2226        let path = format!(
2227            "/worker/registrations/{}",
2228            percent_encode_path_segment(worker_id)
2229        );
2230        self.request_json(
2231            reqwest::Method::DELETE,
2232            &path,
2233            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2234            Option::<&Value>::None,
2235        )
2236        .await
2237    }
2238
2239    /// Long-poll for an ephemeral, read-only workflow query task.
2240    pub async fn poll_query_task(
2241        &self,
2242        worker_id: &str,
2243        task_queue: &str,
2244        timeout: Duration,
2245    ) -> Result<Option<QueryTask>> {
2246        Ok(self
2247            .poll_query_task_response(worker_id, task_queue, timeout)
2248            .await?
2249            .task)
2250    }
2251
2252    /// Poll a query task while preserving server stop and drain metadata.
2253    pub async fn poll_query_task_response(
2254        &self,
2255        worker_id: &str,
2256        task_queue: &str,
2257        timeout: Duration,
2258    ) -> Result<PollQueryTaskResponse> {
2259        let poll_request_id = unique_request_id("rust-query-poll");
2260        self.poll_query_task_response_with_request_id(
2261            worker_id,
2262            task_queue,
2263            timeout,
2264            &poll_request_id,
2265            1,
2266        )
2267        .await
2268    }
2269
2270    async fn poll_query_task_response_with_request_id(
2271        &self,
2272        worker_id: &str,
2273        task_queue: &str,
2274        timeout: Duration,
2275        poll_request_id: &str,
2276        transport_retries: usize,
2277    ) -> Result<PollQueryTaskResponse> {
2278        let timeout_seconds = long_poll_timeout_seconds(timeout);
2279        let body = json!({
2280            "worker_id": worker_id,
2281            "task_queue": task_queue,
2282            "poll_request_id": poll_request_id,
2283            "timeout_seconds": timeout_seconds,
2284        });
2285        self.poll_request_json(
2286            "/worker/query-tasks/poll",
2287            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2288            &body,
2289            timeout + Duration::from_secs(5),
2290            transport_retries,
2291        )
2292        .await
2293    }
2294
2295    /// Complete a query task without appending workflow history.
2296    pub async fn complete_query_task<T: Serialize>(
2297        &self,
2298        query_task_id: &str,
2299        lease_owner: &str,
2300        query_task_attempt: u64,
2301        result: T,
2302        codec: &str,
2303    ) -> Result<Value> {
2304        let typed_result = AvroValue::from_serialize(&result)?;
2305        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2306        self.complete_query_task_with_envelope(
2307            query_task_id,
2308            lease_owner,
2309            query_task_attempt,
2310            typed_result.into_json()?,
2311            result_envelope,
2312        )
2313        .await
2314    }
2315
2316    async fn complete_query_task_with_envelope(
2317        &self,
2318        query_task_id: &str,
2319        lease_owner: &str,
2320        query_task_attempt: u64,
2321        result: Value,
2322        result_envelope: Value,
2323    ) -> Result<Value> {
2324        let body = json!({
2325            "lease_owner": lease_owner,
2326            "query_task_attempt": query_task_attempt,
2327            "result": result,
2328            "result_envelope": result_envelope,
2329        });
2330        let path = format!("/worker/query-tasks/{query_task_id}/complete");
2331        let response = self
2332            .request_json(
2333                reqwest::Method::POST,
2334                &path,
2335                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2336                Some(&body),
2337            )
2338            .await;
2339        query_task_response(response)
2340    }
2341
2342    /// Report a stable machine-readable query-task failure.
2343    pub async fn fail_query_task(
2344        &self,
2345        query_task_id: &str,
2346        lease_owner: &str,
2347        query_task_attempt: u64,
2348        message: impl Into<String>,
2349        reason: impl Into<String>,
2350        failure_type: impl Into<String>,
2351    ) -> Result<Value> {
2352        let body = json!({
2353            "lease_owner": lease_owner,
2354            "query_task_attempt": query_task_attempt,
2355            "failure": {
2356                "message": message.into(),
2357                "reason": reason.into(),
2358                "type": failure_type.into(),
2359            }
2360        });
2361        let path = format!("/worker/query-tasks/{query_task_id}/fail");
2362        let response = self
2363            .request_json(
2364                reqwest::Method::POST,
2365                &path,
2366                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2367                Some(&body),
2368            )
2369            .await;
2370        query_task_response(response)
2371    }
2372
2373    pub async fn heartbeat_worker(
2374        &self,
2375        worker_id: &str,
2376        workflow_available: usize,
2377        activity_available: usize,
2378    ) -> Result<Value> {
2379        let body = json!({
2380            "worker_id": worker_id,
2381            "task_slots": {
2382                "workflow_available": workflow_available,
2383                "activity_available": activity_available
2384            },
2385            "process_metrics": {
2386                "process_id": std::process::id(),
2387                "process_uptime_seconds": 0
2388            }
2389        });
2390
2391        self.request_json(
2392            reqwest::Method::POST,
2393            "/worker/heartbeat",
2394            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2395            Some(&body),
2396        )
2397        .await
2398    }
2399
2400    pub async fn poll_workflow_task(
2401        &self,
2402        worker_id: &str,
2403        task_queue: &str,
2404        timeout: Duration,
2405    ) -> Result<Option<WorkflowTask>> {
2406        Ok(self
2407            .poll_workflow_task_response(worker_id, task_queue, timeout)
2408            .await?
2409            .task)
2410    }
2411
2412    pub async fn poll_workflow_task_response(
2413        &self,
2414        worker_id: &str,
2415        task_queue: &str,
2416        timeout: Duration,
2417    ) -> Result<PollWorkflowTaskResponse> {
2418        let poll_request_id = unique_request_id("rust-workflow-poll");
2419        self.poll_workflow_task_response_with_request_id(
2420            worker_id,
2421            task_queue,
2422            timeout,
2423            &poll_request_id,
2424            1,
2425        )
2426        .await
2427    }
2428
2429    async fn poll_workflow_task_response_with_request_id(
2430        &self,
2431        worker_id: &str,
2432        task_queue: &str,
2433        timeout: Duration,
2434        poll_request_id: &str,
2435        transport_retries: usize,
2436    ) -> Result<PollWorkflowTaskResponse> {
2437        let body = json!({
2438            "worker_id": worker_id,
2439            "task_queue": task_queue,
2440            "poll_request_id": poll_request_id,
2441            "timeout_seconds": long_poll_timeout_seconds(timeout),
2442        });
2443        let mut data: PollWorkflowTaskResponse = self
2444            .poll_request_json(
2445                "/worker/workflow-tasks/poll",
2446                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2447                &body,
2448                timeout + Duration::from_secs(5),
2449                transport_retries,
2450            )
2451            .await?;
2452
2453        if let Some(task) = data.task.as_mut() {
2454            self.fetch_remaining_workflow_history(worker_id, task)
2455                .await?;
2456        }
2457
2458        Ok(data)
2459    }
2460
2461    async fn fetch_remaining_workflow_history(
2462        &self,
2463        worker_id: &str,
2464        task: &mut WorkflowTask,
2465    ) -> Result<()> {
2466        let mut next_token = task.next_history_page_token.clone();
2467
2468        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2469            let lease_owner = task
2470                .lease_owner
2471                .clone()
2472                .unwrap_or_else(|| worker_id.to_string());
2473            let page = self
2474                .workflow_task_history_page(
2475                    &task.task_id,
2476                    &lease_owner,
2477                    task.workflow_task_attempt,
2478                    &token,
2479                )
2480                .await?;
2481
2482            task.append_history_page(page);
2483
2484            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2485                return Err(Error::Codec(
2486                    "workflow history pagination returned the same page token".to_string(),
2487                ));
2488            }
2489
2490            next_token = task.next_history_page_token.clone();
2491        }
2492
2493        Ok(())
2494    }
2495
2496    async fn workflow_task_history_page(
2497        &self,
2498        task_id: &str,
2499        lease_owner: &str,
2500        workflow_task_attempt: u64,
2501        next_history_page_token: &str,
2502    ) -> Result<WorkflowTaskHistoryPage> {
2503        let body = json!({
2504            "lease_owner": lease_owner,
2505            "workflow_task_attempt": workflow_task_attempt,
2506            "next_history_page_token": next_history_page_token
2507        });
2508        let path = format!("/worker/workflow-tasks/{task_id}/history");
2509
2510        self.request_json(
2511            reqwest::Method::POST,
2512            &path,
2513            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2514            Some(&body),
2515        )
2516        .await
2517    }
2518
2519    pub async fn complete_workflow_task(
2520        &self,
2521        task_id: &str,
2522        lease_owner: &str,
2523        workflow_task_attempt: u64,
2524        commands: Vec<Value>,
2525    ) -> Result<Value> {
2526        validate_workflow_task_commands(&commands)?;
2527        let body = json!({
2528            "lease_owner": lease_owner,
2529            "workflow_task_attempt": workflow_task_attempt,
2530            "commands": commands
2531        });
2532        let path = format!("/worker/workflow-tasks/{task_id}/complete");
2533        self.request_json(
2534            reqwest::Method::POST,
2535            &path,
2536            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2537            Some(&body),
2538        )
2539        .await
2540    }
2541
2542    pub async fn fail_workflow_task(
2543        &self,
2544        task_id: &str,
2545        lease_owner: &str,
2546        workflow_task_attempt: u64,
2547        message: impl Into<String>,
2548    ) -> Result<Value> {
2549        self.fail_workflow_task_with_type(
2550            task_id,
2551            lease_owner,
2552            workflow_task_attempt,
2553            message,
2554            "RustWorkflowTaskFailure",
2555        )
2556        .await
2557    }
2558
2559    async fn fail_workflow_task_with_type(
2560        &self,
2561        task_id: &str,
2562        lease_owner: &str,
2563        workflow_task_attempt: u64,
2564        message: impl Into<String>,
2565        failure_type: &str,
2566    ) -> Result<Value> {
2567        let body = json!({
2568            "lease_owner": lease_owner,
2569            "workflow_task_attempt": workflow_task_attempt,
2570            "failure": {
2571                "message": message.into(),
2572                "type": failure_type
2573            }
2574        });
2575        let path = format!("/worker/workflow-tasks/{task_id}/fail");
2576        self.request_json(
2577            reqwest::Method::POST,
2578            &path,
2579            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2580            Some(&body),
2581        )
2582        .await
2583    }
2584
2585    pub async fn poll_activity_task(
2586        &self,
2587        worker_id: &str,
2588        task_queue: &str,
2589        timeout: Duration,
2590    ) -> Result<Option<ActivityTask>> {
2591        Ok(self
2592            .poll_activity_task_response(worker_id, task_queue, timeout)
2593            .await?
2594            .task)
2595    }
2596
2597    /// Poll an activity task while preserving server stop and drain metadata.
2598    pub async fn poll_activity_task_response(
2599        &self,
2600        worker_id: &str,
2601        task_queue: &str,
2602        timeout: Duration,
2603    ) -> Result<PollActivityTaskResponse> {
2604        let poll_request_id = unique_request_id("rust-activity-poll");
2605        self.poll_activity_task_response_with_request_id(
2606            worker_id,
2607            task_queue,
2608            timeout,
2609            &poll_request_id,
2610            1,
2611        )
2612        .await
2613    }
2614
2615    async fn poll_activity_task_response_with_request_id(
2616        &self,
2617        worker_id: &str,
2618        task_queue: &str,
2619        timeout: Duration,
2620        poll_request_id: &str,
2621        transport_retries: usize,
2622    ) -> Result<PollActivityTaskResponse> {
2623        let body = json!({
2624            "worker_id": worker_id,
2625            "task_queue": task_queue,
2626            "poll_request_id": poll_request_id,
2627            "timeout_seconds": long_poll_timeout_seconds(timeout),
2628        });
2629        let data: PollActivityTaskResponse = self
2630            .poll_request_json(
2631                "/worker/activity-tasks/poll",
2632                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2633                &body,
2634                timeout + Duration::from_secs(5),
2635                transport_retries,
2636            )
2637            .await?;
2638        Ok(data)
2639    }
2640
2641    pub async fn complete_activity_task<T: Serialize>(
2642        &self,
2643        task_id: &str,
2644        activity_attempt_id: &str,
2645        lease_owner: &str,
2646        result: T,
2647        codec: &str,
2648    ) -> Result<Value> {
2649        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2650        let body = json!({
2651            "activity_attempt_id": activity_attempt_id,
2652            "lease_owner": lease_owner,
2653            "result": result
2654        });
2655        let path = format!("/worker/activity-tasks/{task_id}/complete");
2656        activity_task_response(
2657            self.request_json(
2658                reqwest::Method::POST,
2659                &path,
2660                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2661                Some(&body),
2662            )
2663            .await,
2664            "complete",
2665            task_id,
2666            activity_attempt_id,
2667        )
2668    }
2669
2670    pub async fn fail_activity_task(
2671        &self,
2672        task_id: &str,
2673        activity_attempt_id: &str,
2674        lease_owner: &str,
2675        message: impl Into<String>,
2676        non_retryable: bool,
2677    ) -> Result<Value> {
2678        let body = json!({
2679            "activity_attempt_id": activity_attempt_id,
2680            "lease_owner": lease_owner,
2681            "failure": {
2682                "message": message.into(),
2683                "type": "RustActivityFailure",
2684                "non_retryable": non_retryable
2685            }
2686        });
2687        let path = format!("/worker/activity-tasks/{task_id}/fail");
2688        activity_task_response(
2689            self.request_json(
2690                reqwest::Method::POST,
2691                &path,
2692                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2693                Some(&body),
2694            )
2695            .await,
2696            "fail",
2697            task_id,
2698            activity_attempt_id,
2699        )
2700    }
2701
2702    pub async fn heartbeat_activity_task<T: Serialize>(
2703        &self,
2704        task_id: &str,
2705        activity_attempt_id: &str,
2706        lease_owner: &str,
2707        details: T,
2708    ) -> Result<ActivityHeartbeatResponse> {
2709        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2710        let body = json!({
2711            "activity_attempt_id": activity_attempt_id,
2712            "lease_owner": lease_owner,
2713            "details": details
2714        });
2715        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2716        activity_task_response(
2717            self.request_json(
2718                reqwest::Method::POST,
2719                &path,
2720                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2721                Some(&body),
2722            )
2723            .await,
2724            "heartbeat",
2725            task_id,
2726            activity_attempt_id,
2727        )
2728    }
2729
2730    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2731        &self,
2732        method: reqwest::Method,
2733        path: &str,
2734        protocol: RequestProtocol,
2735        body: Option<&B>,
2736    ) -> Result<T> {
2737        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2738            .await
2739    }
2740
2741    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2742        &self,
2743        method: reqwest::Method,
2744        path: &str,
2745        protocol: RequestProtocol,
2746        body: Option<&B>,
2747        timeout: Duration,
2748    ) -> Result<T> {
2749        let auth_token = self.auth_token(protocol)?;
2750        let mut request = self
2751            .http
2752            .request(method, format!("{}/api{}", self.base_url, path))
2753            .timeout(timeout)
2754            .header(reqwest::header::ACCEPT, "application/json")
2755            .header(reqwest::header::CONTENT_TYPE, "application/json")
2756            .header("X-Namespace", &self.namespace);
2757
2758        match protocol {
2759            RequestProtocol::Worker(version) => {
2760                request = request.header("X-Durable-Workflow-Protocol-Version", version);
2761            }
2762            RequestProtocol::ControlPlane => {
2763                request = request.header(
2764                    "X-Durable-Workflow-Control-Plane-Version",
2765                    CONTROL_PLANE_VERSION,
2766                );
2767            }
2768        }
2769
2770        if let Some(token) = auth_token {
2771            request = request.bearer_auth(token);
2772        }
2773
2774        if let Some(body) = body {
2775            request = request.json(body);
2776        }
2777
2778        let response = request.send().await?;
2779        let status = response.status();
2780        let bytes = response.bytes().await?;
2781
2782        if !status.is_success() {
2783            let body = String::from_utf8_lossy(&bytes).to_string();
2784            if let Some(protocol) = protocol_failure(status, &body) {
2785                return Err(Error::Protocol(protocol));
2786            }
2787            return Err(Error::Http { status, body });
2788        }
2789
2790        if bytes.is_empty() {
2791            return Ok(serde_json::from_value(Value::Null)?);
2792        }
2793
2794        Ok(serde_json::from_slice(&bytes)?)
2795    }
2796
2797    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2798        &self,
2799        path: &str,
2800        protocol: RequestProtocol,
2801        body: &B,
2802        timeout: Duration,
2803        max_retries: usize,
2804    ) -> Result<T> {
2805        let mut retries = 0;
2806
2807        loop {
2808            let response = self
2809                .request_json_with_timeout(
2810                    reqwest::Method::POST,
2811                    path,
2812                    protocol,
2813                    Some(body),
2814                    timeout,
2815                )
2816                .await;
2817
2818            match response {
2819                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2820                response => return worker_poll_response(response),
2821            }
2822        }
2823    }
2824
2825    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2826        match protocol {
2827            RequestProtocol::Worker(_) => {
2828                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2829                    return Ok(Some(token));
2830                }
2831                if self.control_token.is_some() {
2832                    return Err(Error::MissingRoleCredentials {
2833                        role: "worker",
2834                        opposite_role: "control",
2835                    });
2836                }
2837                Ok(None)
2838            }
2839            RequestProtocol::ControlPlane => {
2840                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2841                    return Ok(Some(token));
2842                }
2843                if self.worker_token.is_some() {
2844                    return Err(Error::MissingRoleCredentials {
2845                        role: "control",
2846                        opposite_role: "worker",
2847                    });
2848                }
2849                Ok(None)
2850            }
2851        }
2852    }
2853}
2854
2855fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2856    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2857    let reason = body
2858        .get("reason")
2859        .and_then(Value::as_str)
2860        .unwrap_or("query_rejected")
2861        .to_string();
2862    let message = body
2863        .get("message")
2864        .or_else(|| body.get("error"))
2865        .and_then(Value::as_str)
2866        .unwrap_or("workflow query was rejected")
2867        .to_string();
2868
2869    QueryFailure {
2870        status: status.as_u16(),
2871        reason,
2872        message,
2873        body,
2874    }
2875}
2876
2877fn workflow_command_result(
2878    command: WorkflowCommandKind,
2879    data: Value,
2880    workflow_id: &str,
2881    run_id: Option<&str>,
2882) -> WorkflowCommandResult {
2883    WorkflowCommandResult {
2884        command,
2885        workflow_id: data
2886            .get("workflow_id")
2887            .and_then(Value::as_str)
2888            .unwrap_or(workflow_id)
2889            .to_string(),
2890        run_id: data
2891            .get("run_id")
2892            .and_then(Value::as_str)
2893            .or(run_id)
2894            .map(str::to_string),
2895        outcome: data
2896            .get("outcome")
2897            .and_then(Value::as_str)
2898            .map(str::to_string),
2899        reason: data
2900            .get("reason")
2901            .and_then(Value::as_str)
2902            .map(str::to_string),
2903        command_status: data
2904            .get("command_status")
2905            .and_then(Value::as_str)
2906            .map(str::to_string),
2907        raw: data,
2908    }
2909}
2910
2911fn workflow_command_rejection(
2912    command: WorkflowCommandKind,
2913    status: reqwest::StatusCode,
2914    raw_body: String,
2915    workflow_id: &str,
2916    run_id: Option<&str>,
2917) -> WorkflowCommandRejection {
2918    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2919    WorkflowCommandRejection {
2920        command,
2921        status: status.as_u16(),
2922        reason: body
2923            .get("reason")
2924            .and_then(Value::as_str)
2925            .unwrap_or("workflow_command_rejected")
2926            .to_string(),
2927        message: body
2928            .get("message")
2929            .or_else(|| body.get("error"))
2930            .and_then(Value::as_str)
2931            .unwrap_or("workflow lifecycle command was rejected")
2932            .to_string(),
2933        workflow_id: body
2934            .get("workflow_id")
2935            .and_then(Value::as_str)
2936            .unwrap_or(workflow_id)
2937            .to_string(),
2938        run_id: body
2939            .get("run_id")
2940            .and_then(Value::as_str)
2941            .or(run_id)
2942            .map(str::to_string),
2943        target_scope: body
2944            .get("target_scope")
2945            .and_then(Value::as_str)
2946            .map(str::to_string),
2947        body,
2948    }
2949}
2950
2951fn query_task_response(response: Result<Value>) -> Result<Value> {
2952    match response {
2953        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2954        response => response,
2955    }
2956}
2957
2958fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2959    match response {
2960        Err(Error::Http { status, body })
2961            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2962        {
2963            Ok(serde_json::from_str(&body)?)
2964        }
2965        response => response,
2966    }
2967}
2968
2969fn worker_poll_body_is_stop(body: &str) -> bool {
2970    serde_json::from_str::<Value>(body)
2971        .ok()
2972        .is_some_and(|body| {
2973            worker_poll_is_stop(
2974                body.get("poll_status").and_then(Value::as_str),
2975                body.get("reason").and_then(Value::as_str),
2976            )
2977        })
2978}
2979
2980fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2981    matches!(poll_status, Some("draining" | "stopped"))
2982        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2983}
2984
2985fn query_task_rejection_is_final(error: &Error) -> bool {
2986    matches!(
2987        error,
2988        Error::QueryFailed(failure)
2989            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2990    )
2991}
2992
2993fn activity_task_response<T>(
2994    response: Result<T>,
2995    operation: &str,
2996    task_id: &str,
2997    activity_attempt_id: &str,
2998) -> Result<T> {
2999    match response {
3000        Err(Error::Http { status, body }) => {
3001            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
3002            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
3003                operation: operation.to_string(),
3004                status: status.as_u16(),
3005                reason: body
3006                    .get("reason")
3007                    .and_then(Value::as_str)
3008                    .unwrap_or("activity_task_rejected")
3009                    .to_string(),
3010                task_id: body
3011                    .get("task_id")
3012                    .and_then(Value::as_str)
3013                    .unwrap_or(task_id)
3014                    .to_string(),
3015                activity_attempt_id: body
3016                    .get("activity_attempt_id")
3017                    .and_then(Value::as_str)
3018                    .unwrap_or(activity_attempt_id)
3019                    .to_string(),
3020                cancel_requested: body
3021                    .get("cancel_requested")
3022                    .and_then(Value::as_bool)
3023                    .unwrap_or(false),
3024                can_continue: body.get("can_continue").and_then(Value::as_bool),
3025                run_closed_reason: body
3026                    .get("run_closed_reason")
3027                    .and_then(Value::as_str)
3028                    .map(str::to_string),
3029                body,
3030            }))
3031        }
3032        response => response,
3033    }
3034}
3035
3036fn activity_task_rejection_is_final(error: &Error) -> bool {
3037    matches!(
3038        error,
3039        Error::ActivityTaskRejected(rejection)
3040            if matches!(
3041                rejection.reason.as_str(),
3042                "run_cancelled"
3043                    | "run_terminated"
3044                    | "attempt_closed"
3045                    | "stale_attempt"
3046                    | "activity_cancelled"
3047                    | "task_cancelled"
3048                    | "run_closed"
3049                    | "activity_not_running"
3050                    | "attempt_not_found"
3051            )
3052    )
3053}
3054
3055fn workflow_task_completion_is_terminal_timeout(
3056    error: &Error,
3057    task_id: &str,
3058    workflow_task_attempt: u64,
3059    run_id: Option<&str>,
3060) -> bool {
3061    let Error::Http { status, body } = error else {
3062        return false;
3063    };
3064    if *status != reqwest::StatusCode::CONFLICT {
3065        return false;
3066    }
3067
3068    let Some(run_id) = run_id else {
3069        return false;
3070    };
3071    let Ok(body) = serde_json::from_str::<Value>(body) else {
3072        return false;
3073    };
3074
3075    body.get("recorded").and_then(Value::as_bool) == Some(false)
3076        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
3077        && body.get("run_status").and_then(Value::as_str) == Some("failed")
3078        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
3079        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
3080        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
3081}
3082
3083fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
3084    let body: Value = serde_json::from_str(raw_body).ok()?;
3085    let reason = body.get("reason")?.as_str()?;
3086    if !matches!(
3087        reason,
3088        "missing_protocol_version"
3089            | "unsupported_protocol_version"
3090            | "missing_control_plane_version"
3091            | "unsupported_control_plane_version"
3092    ) {
3093        return None;
3094    }
3095
3096    Some(ProtocolFailure {
3097        status: status.as_u16(),
3098        reason: reason.to_string(),
3099        message: body
3100            .get("message")
3101            .or_else(|| body.get("error"))
3102            .and_then(Value::as_str)
3103            .unwrap_or("protocol version rejected")
3104            .to_string(),
3105        supported_version: body
3106            .get("supported_version")
3107            .and_then(Value::as_str)
3108            .map(str::to_string),
3109        requested_version: body
3110            .get("requested_version")
3111            .and_then(Value::as_str)
3112            .map(str::to_string),
3113        body,
3114    })
3115}
3116
3117fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3118    timeout
3119        .as_secs()
3120        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3121        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3122}
3123
3124fn worker_operation_is_retryable(error: &Error) -> bool {
3125    match error {
3126        Error::Transport(error) => {
3127            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3128        }
3129        Error::Http { status, .. } => {
3130            matches!(
3131                *status,
3132                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3133            ) || status.is_server_error()
3134        }
3135        _ => false,
3136    }
3137}
3138
3139fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3140    let exponent = retry.saturating_sub(1).min(31) as u32;
3141    policy
3142        .initial_backoff
3143        .saturating_mul(1_u32 << exponent)
3144        .min(policy.max_backoff)
3145}
3146
3147#[derive(Debug)]
3148pub struct ClientBuilder {
3149    base_url: String,
3150    token: Option<String>,
3151    control_token: Option<String>,
3152    worker_token: Option<String>,
3153    namespace: String,
3154    timeout: Duration,
3155}
3156
3157impl ClientBuilder {
3158    pub fn token(mut self, token: Option<String>) -> Self {
3159        self.token = token;
3160        self
3161    }
3162
3163    pub fn control_token(mut self, token: Option<String>) -> Self {
3164        self.control_token = token;
3165        self
3166    }
3167
3168    pub fn worker_token(mut self, token: Option<String>) -> Self {
3169        self.worker_token = token;
3170        self
3171    }
3172
3173    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3174        self.namespace = namespace.into();
3175        self
3176    }
3177
3178    pub fn timeout(mut self, timeout: Duration) -> Self {
3179        self.timeout = timeout;
3180        self
3181    }
3182
3183    pub fn build(self) -> Result<Client> {
3184        let base_url = self.base_url.trim_end_matches('/').to_string();
3185        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3186            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3187            .unwrap_or_else(|_| base_url.ends_with("/api"));
3188
3189        if has_sdk_api_suffix {
3190            return Err(Error::InvalidBaseUrl);
3191        }
3192
3193        Ok(Client {
3194            http: reqwest::Client::builder().timeout(self.timeout).build()?,
3195            base_url,
3196            token: self.token,
3197            control_token: self.control_token,
3198            worker_token: self.worker_token,
3199            namespace: self.namespace,
3200        })
3201    }
3202}
3203
3204#[derive(Clone, Debug)]
3205pub struct WorkflowHandle {
3206    client: Client,
3207    pub workflow_id: String,
3208    pub run_id: Option<String>,
3209    pub workflow_type: String,
3210}
3211
3212impl WorkflowHandle {
3213    /// Describe whichever run is current for this stable workflow instance.
3214    pub async fn describe(&self) -> Result<WorkflowDescription> {
3215        self.client.describe_workflow(&self.workflow_id).await
3216    }
3217
3218    /// Describe the run identity originally selected by this handle.
3219    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3220        let run_id = self.run_id.as_deref().ok_or_else(|| {
3221            Error::Codec("run_id is required for selected-run description".to_string())
3222        })?;
3223        self.client
3224            .describe_workflow_run(&self.workflow_id, run_id)
3225            .await
3226    }
3227
3228    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3229        self.client
3230            .signal_workflow(&self.workflow_id, signal_name, input)
3231            .await
3232    }
3233
3234    /// Signal only if this handle's selected run is still current.
3235    pub async fn signal_selected_run<T: Serialize>(
3236        &self,
3237        signal_name: &str,
3238        input: T,
3239    ) -> Result<Value> {
3240        let run_id = self.run_id.as_deref().ok_or_else(|| {
3241            Error::Codec("run_id is required for selected-run signaling".to_string())
3242        })?;
3243        self.client
3244            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3245            .await
3246    }
3247
3248    /// Request cooperative cancellation of whichever run is current.
3249    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3250        self.client
3251            .cancel_workflow(&self.workflow_id, options)
3252            .await
3253    }
3254
3255    /// Request cancellation only if this handle's selected run is still current.
3256    pub async fn cancel_selected_run(
3257        &self,
3258        options: WorkflowCommandOptions,
3259    ) -> Result<WorkflowCommandResult> {
3260        let run_id = self.run_id.as_deref().ok_or_else(|| {
3261            Error::Codec("run_id is required for selected-run cancellation".to_string())
3262        })?;
3263        self.client
3264            .cancel_workflow_run(&self.workflow_id, run_id, options)
3265            .await
3266    }
3267
3268    /// Forcefully terminate whichever run is current.
3269    pub async fn terminate(
3270        &self,
3271        options: WorkflowCommandOptions,
3272    ) -> Result<WorkflowCommandResult> {
3273        self.client
3274            .terminate_workflow(&self.workflow_id, options)
3275            .await
3276    }
3277
3278    /// Terminate only if this handle's selected run is still current.
3279    pub async fn terminate_selected_run(
3280        &self,
3281        options: WorkflowCommandOptions,
3282    ) -> Result<WorkflowCommandResult> {
3283        let run_id = self.run_id.as_deref().ok_or_else(|| {
3284            Error::Codec("run_id is required for selected-run termination".to_string())
3285        })?;
3286        self.client
3287            .terminate_workflow_run(&self.workflow_id, run_id, options)
3288            .await
3289    }
3290
3291    /// Execute a named, read-only query against this workflow.
3292    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3293        self.client
3294            .query_workflow(&self.workflow_id, query_name, input)
3295            .await
3296    }
3297
3298    pub async fn query_avro_value<T: Serialize>(
3299        &self,
3300        query_name: &str,
3301        input: T,
3302    ) -> Result<AvroValue> {
3303        self.client
3304            .query_workflow_avro_value(&self.workflow_id, query_name, input)
3305            .await
3306    }
3307
3308    pub async fn update<T: Serialize>(
3309        &self,
3310        update_name: &str,
3311        input: T,
3312        request_id: Option<&str>,
3313    ) -> Result<Value> {
3314        self.client
3315            .update_workflow(&self.workflow_id, update_name, input, request_id)
3316            .await
3317    }
3318
3319    pub async fn update_avro_value<T: Serialize>(
3320        &self,
3321        update_name: &str,
3322        input: T,
3323        request_id: Option<&str>,
3324    ) -> Result<AvroValue> {
3325        self.client
3326            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3327            .await
3328    }
3329
3330    /// Query only if this handle's selected run is still current.
3331    pub async fn query_selected_run<T: Serialize>(
3332        &self,
3333        query_name: &str,
3334        input: T,
3335    ) -> Result<Value> {
3336        let run_id = self
3337            .run_id
3338            .as_deref()
3339            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3340        self.client
3341            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3342            .await
3343    }
3344
3345    /// Await the final terminal outcome of the current continue-as-new chain.
3346    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3347        self.result_target(options, None).await
3348    }
3349
3350    /// Await the final result without projecting Avro bytes through JSON.
3351    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3352        self.result_avro_value_target(options, None).await
3353    }
3354
3355    /// Await only the run identity originally selected by this handle.
3356    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3357        let run_id = self.run_id.as_deref().ok_or_else(|| {
3358            Error::Codec("run_id is required for selected-run result".to_string())
3359        })?;
3360        self.result_target(options, Some(run_id)).await
3361    }
3362
3363    /// Await the selected run's result on the lossless Avro Value surface.
3364    pub async fn result_selected_run_avro_value(
3365        &self,
3366        options: WorkflowResultOptions,
3367    ) -> Result<AvroValue> {
3368        let run_id = self.run_id.as_deref().ok_or_else(|| {
3369            Error::Codec("run_id is required for selected-run result".to_string())
3370        })?;
3371        self.result_avro_value_target(options, Some(run_id)).await
3372    }
3373
3374    async fn result_avro_value_target(
3375        &self,
3376        options: WorkflowResultOptions,
3377        selected_run_id: Option<&str>,
3378    ) -> Result<AvroValue> {
3379        let started = Instant::now();
3380
3381        loop {
3382            let description = match selected_run_id {
3383                Some(run_id) => {
3384                    self.client
3385                        .describe_workflow_run(&self.workflow_id, run_id)
3386                        .await?
3387                }
3388                None => self.describe().await?,
3389            };
3390            if description.is_completed() {
3391                return description.output_avro_value.ok_or_else(|| {
3392                    Error::Codec(
3393                        "missing_payload_envelope: typed workflow result requires output_envelope"
3394                            .to_string(),
3395                    )
3396                });
3397            }
3398            if description.is_terminal() {
3399                let outcome =
3400                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3401                return Err(match outcome.kind {
3402                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3403                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3404                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3405                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3406                });
3407            }
3408            if started.elapsed() >= options.timeout {
3409                return Err(Error::Timeout);
3410            }
3411            tokio::time::sleep(options.poll_interval).await;
3412        }
3413    }
3414
3415    async fn result_target(
3416        &self,
3417        options: WorkflowResultOptions,
3418        selected_run_id: Option<&str>,
3419    ) -> Result<Value> {
3420        let started = Instant::now();
3421
3422        loop {
3423            let description = match selected_run_id {
3424                Some(run_id) => {
3425                    self.client
3426                        .describe_workflow_run(&self.workflow_id, run_id)
3427                        .await?
3428                }
3429                None => self.describe().await?,
3430            };
3431            if description.is_completed() {
3432                return Ok(description.output.unwrap_or(Value::Null));
3433            }
3434
3435            if description.is_terminal() {
3436                let outcome =
3437                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3438                return Err(match outcome.kind {
3439                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3440                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3441                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3442                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3443                });
3444            }
3445
3446            if started.elapsed() >= options.timeout {
3447                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3448                    kind: WorkflowTerminalKind::TimedOut,
3449                    workflow_id: description
3450                        .workflow_id
3451                        .clone()
3452                        .unwrap_or_else(|| self.workflow_id.clone()),
3453                    run_id: description
3454                        .run_id
3455                        .clone()
3456                        .or_else(|| selected_run_id.map(str::to_string)),
3457                    reason: "result_wait_timeout".to_string(),
3458                    failure_category: Some("client_timeout".to_string()),
3459                    failure_id: None,
3460                    exception_type: None,
3461                    exception_class: None,
3462                    non_retryable: None,
3463                    message: Some(format!(
3464                        "workflow result was not terminal within {:?}",
3465                        options.timeout
3466                    )),
3467                    exception: None,
3468                    raw: description.raw_value(),
3469                }));
3470            }
3471
3472            tokio::time::sleep(options.poll_interval).await;
3473        }
3474    }
3475}
3476
3477#[derive(Clone, Copy, Debug)]
3478pub struct WorkflowResultOptions {
3479    pub poll_interval: Duration,
3480    pub timeout: Duration,
3481}
3482
3483impl Default for WorkflowResultOptions {
3484    fn default() -> Self {
3485        Self {
3486            poll_interval: Duration::from_millis(500),
3487            timeout: Duration::from_secs(30),
3488        }
3489    }
3490}
3491
3492#[derive(Clone, Debug, Deserialize)]
3493pub struct WorkflowDescription {
3494    pub workflow_id: Option<String>,
3495    pub run_id: Option<String>,
3496    pub workflow_type: Option<String>,
3497    pub status: Option<String>,
3498    #[serde(default)]
3499    pub closed_reason: Option<String>,
3500    #[serde(default)]
3501    pub error: Option<String>,
3502    #[serde(default)]
3503    pub failure: Option<Value>,
3504    #[serde(default)]
3505    pub exception: Option<Value>,
3506    #[serde(default)]
3507    pub failures: Vec<Value>,
3508    #[serde(default)]
3509    pub output: Option<Value>,
3510    #[serde(default)]
3511    pub output_envelope: Option<Value>,
3512    #[serde(skip)]
3513    pub output_avro_value: Option<AvroValue>,
3514    #[serde(flatten)]
3515    pub raw: HashMap<String, Value>,
3516}
3517
3518impl WorkflowDescription {
3519    pub fn is_completed(&self) -> bool {
3520        matches!(self.status.as_deref(), Some("completed" | "Completed"))
3521    }
3522
3523    pub fn is_terminal(&self) -> bool {
3524        matches!(
3525            self.status.as_deref(),
3526            Some(
3527                "completed"
3528                    | "Completed"
3529                    | "failed"
3530                    | "Failed"
3531                    | "cancelled"
3532                    | "Cancelled"
3533                    | "terminated"
3534                    | "Terminated"
3535                    | "timed_out"
3536                    | "TimedOut",
3537            )
3538        )
3539    }
3540
3541    fn decode_payloads(&mut self) -> Result<()> {
3542        if let Some(envelope) = &self.output_envelope {
3543            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3544            self.output = Some(value.clone().into_json()?);
3545            self.output_avro_value = Some(value);
3546        }
3547
3548        Ok(())
3549    }
3550
3551    fn raw_value(&self) -> Value {
3552        let mut data = self.raw.clone();
3553        data.insert(
3554            "workflow_id".to_string(),
3555            self.workflow_id
3556                .clone()
3557                .map(Value::String)
3558                .unwrap_or(Value::Null),
3559        );
3560        data.insert(
3561            "run_id".to_string(),
3562            self.run_id
3563                .clone()
3564                .map(Value::String)
3565                .unwrap_or(Value::Null),
3566        );
3567        data.insert(
3568            "workflow_type".to_string(),
3569            self.workflow_type
3570                .clone()
3571                .map(Value::String)
3572                .unwrap_or(Value::Null),
3573        );
3574        data.insert(
3575            "status".to_string(),
3576            self.status
3577                .clone()
3578                .map(Value::String)
3579                .unwrap_or(Value::Null),
3580        );
3581        data.insert(
3582            "closed_reason".to_string(),
3583            self.closed_reason
3584                .clone()
3585                .map(Value::String)
3586                .unwrap_or(Value::Null),
3587        );
3588        if let Some(failure) = &self.failure {
3589            data.insert("failure".to_string(), failure.clone());
3590        }
3591        if let Some(exception) = &self.exception {
3592            data.insert("exception".to_string(), exception.clone());
3593        }
3594        Value::Object(data.into_iter().collect())
3595    }
3596}
3597
3598fn workflow_terminal_outcome(
3599    description: &WorkflowDescription,
3600    workflow_id: &str,
3601    run_id: Option<&str>,
3602) -> WorkflowTerminalOutcome {
3603    let terminal_kind = description
3604        .closed_reason
3605        .as_deref()
3606        .or(description.status.as_deref())
3607        .unwrap_or("failed")
3608        .to_ascii_lowercase();
3609    let kind = match terminal_kind.as_str() {
3610        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3611        "terminated" => WorkflowTerminalKind::Terminated,
3612        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3613        _ => WorkflowTerminalKind::Failed,
3614    };
3615    let default_reason = match kind {
3616        WorkflowTerminalKind::Failed => "workflow_failed",
3617        WorkflowTerminalKind::Cancelled => "cancelled",
3618        WorkflowTerminalKind::Terminated => "terminated",
3619        WorkflowTerminalKind::TimedOut => "timed_out",
3620    };
3621    let failure = description
3622        .failure
3623        .as_ref()
3624        .filter(|value| value.is_object());
3625    let nested_failure = failure
3626        .and_then(|value| value.get("failures"))
3627        .and_then(Value::as_array)
3628        .and_then(|failures| failures.last())
3629        .or_else(|| description.failures.last());
3630    let exception = description
3631        .exception
3632        .clone()
3633        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3634        .or_else(|| {
3635            nested_failure
3636                .and_then(|value| value.get("exception_payload"))
3637                .cloned()
3638        });
3639    let string_field = |name: &str| {
3640        failure
3641            .and_then(|value| value.get(name))
3642            .and_then(Value::as_str)
3643            .or_else(|| {
3644                nested_failure
3645                    .and_then(|value| value.get(name))
3646                    .and_then(Value::as_str)
3647            })
3648            .map(str::to_string)
3649    };
3650    let exception_field = |name: &str| {
3651        exception
3652            .as_ref()
3653            .and_then(|value| value.get(name))
3654            .and_then(Value::as_str)
3655            .map(str::to_string)
3656    };
3657    let message = description
3658        .error
3659        .clone()
3660        .or_else(|| string_field("message"))
3661        .or_else(|| exception_field("message"));
3662    let reason = description
3663        .raw
3664        .get("reason")
3665        .and_then(Value::as_str)
3666        .map(str::to_string)
3667        .or_else(|| {
3668            failure
3669                .and_then(|value| value.get("reason"))
3670                .and_then(Value::as_str)
3671                .map(str::to_string)
3672        })
3673        .or_else(|| description.closed_reason.clone())
3674        .unwrap_or_else(|| default_reason.to_string());
3675    let failure_id = string_field("failure_id").or_else(|| {
3676        nested_failure
3677            .and_then(|value| value.get("id"))
3678            .and_then(Value::as_str)
3679            .map(str::to_string)
3680    });
3681
3682    WorkflowTerminalOutcome {
3683        kind,
3684        workflow_id: description
3685            .workflow_id
3686            .clone()
3687            .unwrap_or_else(|| workflow_id.to_string()),
3688        run_id: description
3689            .run_id
3690            .clone()
3691            .or_else(|| run_id.map(str::to_string)),
3692        reason,
3693        failure_category: string_field("failure_category")
3694            .or_else(|| Some(default_reason.to_string())),
3695        failure_id,
3696        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3697        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3698        non_retryable: failure
3699            .and_then(|value| value.get("non_retryable"))
3700            .and_then(Value::as_bool)
3701            .or_else(|| {
3702                nested_failure
3703                    .and_then(|value| value.get("non_retryable"))
3704                    .and_then(Value::as_bool)
3705            }),
3706        message,
3707        exception,
3708        raw: description.raw_value(),
3709    }
3710}
3711
3712#[derive(Clone, Debug, Deserialize)]
3713pub struct RegisterWorkerResponse {
3714    pub worker_id: String,
3715    pub registered: bool,
3716    #[serde(default)]
3717    pub heartbeat_interval_seconds: Option<u64>,
3718    #[serde(default)]
3719    pub protocol_version: Option<String>,
3720    #[serde(default)]
3721    pub server_capabilities: Option<Value>,
3722}
3723
3724/// Result of gracefully removing a worker-plane registration.
3725#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3726pub struct WorkerDeregistrationEnvelope {
3727    pub worker_id: String,
3728    pub outcome: String,
3729    pub recovered_workflow_task_count: u64,
3730}
3731
3732#[derive(Clone, Debug, Deserialize)]
3733pub struct PollWorkflowTaskResponse {
3734    #[serde(default)]
3735    pub task: Option<WorkflowTask>,
3736    #[serde(default)]
3737    pub poll_status: Option<String>,
3738    #[serde(default)]
3739    pub reason: Option<String>,
3740    #[serde(default)]
3741    pub protocol_version: Option<String>,
3742    #[serde(default)]
3743    pub server_capabilities: Option<Value>,
3744}
3745
3746impl PollWorkflowTaskResponse {
3747    /// Classify this response without parsing server display text.
3748    pub fn outcome(&self) -> WorkerPollOutcome {
3749        worker_poll_outcome(
3750            self.task.is_some(),
3751            self.poll_status.as_deref(),
3752            self.reason.as_deref(),
3753        )
3754    }
3755}
3756
3757#[derive(Clone, Debug, Deserialize)]
3758pub struct PollActivityTaskResponse {
3759    #[serde(default)]
3760    pub task: Option<ActivityTask>,
3761    #[serde(default)]
3762    pub poll_status: Option<String>,
3763    #[serde(default)]
3764    pub reason: Option<String>,
3765}
3766
3767impl PollActivityTaskResponse {
3768    /// Classify this response without parsing server display text.
3769    pub fn outcome(&self) -> WorkerPollOutcome {
3770        worker_poll_outcome(
3771            self.task.is_some(),
3772            self.poll_status.as_deref(),
3773            self.reason.as_deref(),
3774        )
3775    }
3776}
3777
3778#[derive(Clone, Debug, Deserialize)]
3779pub struct PollQueryTaskResponse {
3780    #[serde(default)]
3781    pub task: Option<QueryTask>,
3782    #[serde(default)]
3783    pub poll_status: Option<String>,
3784    #[serde(default)]
3785    pub reason: Option<String>,
3786}
3787
3788impl PollQueryTaskResponse {
3789    /// Classify this response without parsing server display text.
3790    pub fn outcome(&self) -> WorkerPollOutcome {
3791        worker_poll_outcome(
3792            self.task.is_some(),
3793            self.poll_status.as_deref(),
3794            self.reason.as_deref(),
3795        )
3796    }
3797}
3798
3799/// Stable classification for worker poll responses.
3800#[derive(Clone, Debug, PartialEq, Eq)]
3801pub enum WorkerPollOutcome {
3802    /// A task was leased and is available on the response.
3803    Task,
3804    /// No task was leased, but the worker should continue polling.
3805    Idle {
3806        poll_status: Option<String>,
3807        reason: Option<String>,
3808    },
3809    /// The server asked this worker to stop claiming new work.
3810    Stop {
3811        poll_status: Option<String>,
3812        reason: Option<String>,
3813    },
3814}
3815
3816impl WorkerPollOutcome {
3817    pub fn should_stop(&self) -> bool {
3818        matches!(self, Self::Stop { .. })
3819    }
3820}
3821
3822fn worker_poll_outcome(
3823    has_task: bool,
3824    poll_status: Option<&str>,
3825    reason: Option<&str>,
3826) -> WorkerPollOutcome {
3827    if worker_poll_is_stop(poll_status, reason) {
3828        return WorkerPollOutcome::Stop {
3829            poll_status: poll_status.map(str::to_string),
3830            reason: reason.map(str::to_string),
3831        };
3832    }
3833
3834    if has_task {
3835        WorkerPollOutcome::Task
3836    } else {
3837        WorkerPollOutcome::Idle {
3838            poll_status: poll_status.map(str::to_string),
3839            reason: reason.map(str::to_string),
3840        }
3841    }
3842}
3843
3844/// An ephemeral server-routed query task.
3845#[derive(Clone, Debug, Deserialize)]
3846pub struct QueryTask {
3847    pub query_task_id: String,
3848    #[serde(default = "default_workflow_task_attempt")]
3849    pub query_task_attempt: u64,
3850    #[serde(default)]
3851    pub lease_owner: Option<String>,
3852    #[serde(default)]
3853    pub workflow_id: Option<String>,
3854    #[serde(default)]
3855    pub run_id: Option<String>,
3856    pub workflow_type: String,
3857    pub query_name: String,
3858    #[serde(default = "default_payload_codec")]
3859    pub payload_codec: String,
3860    #[serde(default)]
3861    pub workflow_arguments: Option<Value>,
3862    #[serde(default)]
3863    pub query_arguments: Option<Value>,
3864    #[serde(default)]
3865    pub history_events: Vec<HistoryEvent>,
3866    #[serde(default)]
3867    pub history_export: Option<Value>,
3868    #[serde(default)]
3869    pub run_status: Option<String>,
3870}
3871
3872#[derive(Clone, Debug, Deserialize)]
3873pub struct WorkflowTask {
3874    pub task_id: String,
3875    #[serde(default)]
3876    pub workflow_id: Option<String>,
3877    #[serde(default)]
3878    pub run_id: Option<String>,
3879    pub workflow_type: String,
3880    #[serde(default = "default_payload_codec")]
3881    pub payload_codec: String,
3882    #[serde(default)]
3883    pub arguments: Option<Value>,
3884    #[serde(default)]
3885    pub history_events: Vec<HistoryEvent>,
3886    #[serde(default)]
3887    pub total_history_events: Option<u64>,
3888    #[serde(default)]
3889    pub history_size_bytes: Option<u64>,
3890    #[serde(default)]
3891    pub continue_as_new_recommended: Option<bool>,
3892    #[serde(default)]
3893    pub history_budget_pressure: Option<String>,
3894    #[serde(default)]
3895    pub next_history_page_token: Option<String>,
3896    #[serde(default = "default_workflow_task_attempt")]
3897    pub workflow_task_attempt: u64,
3898    #[serde(default)]
3899    pub workflow_signal_id: Option<String>,
3900    #[serde(default)]
3901    pub signal_name: Option<String>,
3902    #[serde(default)]
3903    pub signal_arguments: Option<Value>,
3904    #[serde(default)]
3905    pub workflow_update_id: Option<String>,
3906    #[serde(default)]
3907    pub update_name: Option<String>,
3908    #[serde(default)]
3909    pub lease_owner: Option<String>,
3910}
3911
3912impl WorkflowTask {
3913    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3914        self.history_events.extend(page.history_events);
3915
3916        if page.total_history_events.is_some() {
3917            self.total_history_events = page.total_history_events;
3918        }
3919
3920        self.next_history_page_token = page
3921            .next_history_page_token
3922            .filter(|token| !token.is_empty());
3923    }
3924}
3925
3926#[derive(Clone, Debug, Deserialize)]
3927struct WorkflowTaskHistoryPage {
3928    #[serde(default)]
3929    history_events: Vec<HistoryEvent>,
3930    #[serde(default)]
3931    total_history_events: Option<u64>,
3932    #[serde(default)]
3933    next_history_page_token: Option<String>,
3934}
3935
3936#[derive(Clone, Debug, Deserialize)]
3937pub struct ActivityTask {
3938    pub task_id: String,
3939    #[serde(default)]
3940    pub activity_attempt_id: Option<String>,
3941    #[serde(default)]
3942    pub attempt_id: Option<String>,
3943    pub activity_type: String,
3944    #[serde(default = "default_payload_codec")]
3945    pub payload_codec: String,
3946    #[serde(default)]
3947    pub arguments: Option<Value>,
3948    #[serde(default = "default_attempt_number")]
3949    pub attempt_number: u64,
3950    #[serde(default)]
3951    pub lease_owner: Option<String>,
3952}
3953
3954#[derive(Clone, Debug, Deserialize)]
3955pub struct HistoryEvent {
3956    #[serde(alias = "type")]
3957    pub event_type: String,
3958    #[serde(default)]
3959    pub payload: Value,
3960    #[serde(flatten)]
3961    pub raw: HashMap<String, Value>,
3962}
3963
3964/// One decoded signal in the committed workflow-history snapshot.
3965#[derive(Clone, Debug, PartialEq)]
3966pub struct QuerySignal {
3967    pub id: Option<String>,
3968    pub name: String,
3969    pub arguments: Vec<Value>,
3970    avro_arguments: Vec<AvroValue>,
3971    pub workflow_sequence: Option<u64>,
3972}
3973
3974impl QuerySignal {
3975    /// Lossless fixed Avro Value arguments for this committed signal.
3976    pub fn arguments_avro_value(&self) -> &[AvroValue] {
3977        &self.avro_arguments
3978    }
3979}
3980
3981/// Immutable state supplied to a registered query handler.
3982///
3983/// This context intentionally exposes no activity, signal-wait, or command
3984/// APIs. Query handlers inspect committed history and return a value; query
3985/// completion does not append an event or advance deterministic execution.
3986#[derive(Clone, Debug)]
3987pub struct QueryContext {
3988    pub workflow_id: Option<String>,
3989    pub run_id: Option<String>,
3990    pub workflow_type: String,
3991    pub run_status: Option<String>,
3992    workflow_input: Value,
3993    workflow_input_avro_value: AvroValue,
3994    history_events: Arc<Vec<HistoryEvent>>,
3995    signal_events: Arc<Vec<QuerySignal>>,
3996}
3997
3998impl QueryContext {
3999    /// The normalized argument list used to start the workflow.
4000    pub fn workflow_input(&self) -> &Value {
4001        &self.workflow_input
4002    }
4003
4004    /// The lossless fixed Avro Value argument list used to start the workflow.
4005    pub fn workflow_input_avro_value(&self) -> &AvroValue {
4006        &self.workflow_input_avro_value
4007    }
4008
4009    /// The immutable committed history used for this query snapshot.
4010    pub fn history_events(&self) -> &[HistoryEvent] {
4011        self.history_events.as_slice()
4012    }
4013
4014    /// All decoded signals in committed workflow order.
4015    pub fn signal_events(&self) -> &[QuerySignal] {
4016        self.signal_events.as_slice()
4017    }
4018
4019    /// Decoded argument lists for each committed signal with `signal_name`.
4020    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
4021        self.signal_events
4022            .iter()
4023            .filter(|signal| signal.name == signal_name)
4024            .map(|signal| signal.arguments.clone())
4025            .collect()
4026    }
4027
4028    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
4029    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
4030        self.signal_events
4031            .iter()
4032            .filter(|signal| signal.name == signal_name)
4033            .map(|signal| signal.avro_arguments.clone())
4034            .collect()
4035    }
4036}
4037
4038#[derive(Clone, Debug, Deserialize)]
4039pub struct ActivityHeartbeatResponse {
4040    #[serde(default)]
4041    pub cancel_requested: bool,
4042    #[serde(default)]
4043    pub heartbeat_recorded: bool,
4044    #[serde(default)]
4045    pub can_continue: Option<bool>,
4046    #[serde(default)]
4047    pub reason: Option<String>,
4048    #[serde(default)]
4049    pub run_closed_reason: Option<String>,
4050    #[serde(default)]
4051    pub run_closed_at: Option<String>,
4052    #[serde(default)]
4053    pub lease_expires_at: Option<String>,
4054    #[serde(default)]
4055    pub last_heartbeat_at: Option<String>,
4056}
4057
4058impl ActivityHeartbeatResponse {
4059    /// Whether the activity should stop instead of attempting completion.
4060    pub fn should_stop(&self) -> bool {
4061        self.cancel_requested || self.can_continue == Some(false)
4062    }
4063}
4064
4065fn default_payload_codec() -> String {
4066    DEFAULT_CODEC.to_string()
4067}
4068
4069fn default_workflow_task_attempt() -> u64 {
4070    1
4071}
4072
4073fn default_attempt_number() -> u64 {
4074    1
4075}
4076
4077type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4078type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
4079type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
4080type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
4081type ReplayedWorkflowHandler =
4082    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
4083type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4084type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
4085type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4086type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4087type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4088type ReplayedQueryHandler = Arc<
4089    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
4090        + Send
4091        + Sync,
4092>;
4093type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
4094
4095struct ReplayedWorkflowInvocation {
4096    future: WorkflowFuture,
4097    snapshot: WorkflowStateSnapshot,
4098}
4099
4100#[derive(Clone)]
4101struct RegisteredWorkflow {
4102    execute: WorkflowHandler,
4103    replay: Option<ReplayedWorkflowHandler>,
4104    state_type: Option<TypeId>,
4105}
4106
4107#[derive(Clone)]
4108enum RegisteredQuery {
4109    Snapshot(QueryHandler),
4110    Replayed {
4111        state_type: TypeId,
4112        handler: ReplayedQueryHandler,
4113    },
4114}
4115
4116#[derive(Clone, Debug)]
4117pub struct WorkerHeartbeatObservation {
4118    pub worker_id: String,
4119    pub task_queue: String,
4120    pub acknowledged_at_unix_millis: u64,
4121    pub acknowledgement: Value,
4122}
4123
4124/// Bounded retry policy for worker poll acquisition and worker heartbeats.
4125///
4126/// Expected empty long polls are normal successful responses. Transport
4127/// failures, HTTP 408/429 responses, and server errors are retried with capped
4128/// exponential backoff. Authentication, protocol, codec, and handler failures
4129/// are never retried by the worker.
4130#[derive(Clone, Copy, Debug)]
4131pub struct WorkerRetryPolicy {
4132    /// Number of retries after the initial request fails.
4133    pub max_retries: usize,
4134    /// Delay before the first retry.
4135    pub initial_backoff: Duration,
4136    /// Maximum delay between retries.
4137    pub max_backoff: Duration,
4138}
4139
4140impl Default for WorkerRetryPolicy {
4141    fn default() -> Self {
4142        Self {
4143            max_retries: 5,
4144            initial_backoff: Duration::from_millis(100),
4145            max_backoff: Duration::from_secs(5),
4146        }
4147    }
4148}
4149
4150#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4151enum ManagedPollOutcome {
4152    Idle,
4153    Handled,
4154    Stop,
4155}
4156
4157#[derive(Clone)]
4158pub struct Worker {
4159    client: Client,
4160    worker_id: String,
4161    task_queue: String,
4162    workflows: HashMap<String, RegisteredWorkflow>,
4163    activities: HashMap<String, ActivityHandler>,
4164    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4165    updates: HashMap<String, HashMap<String, UpdateHandler>>,
4166    max_concurrent_workflow_tasks: usize,
4167    max_concurrent_activity_tasks: usize,
4168    poll_timeout: Duration,
4169    heartbeat_interval: Duration,
4170    retry_policy: WorkerRetryPolicy,
4171    heartbeat_observer: Option<WorkerHeartbeatObserver>,
4172}
4173
4174impl Worker {
4175    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4176        Self {
4177            client,
4178            worker_id: default_worker_id(),
4179            task_queue: task_queue.into(),
4180            workflows: HashMap::new(),
4181            activities: HashMap::new(),
4182            queries: HashMap::new(),
4183            updates: HashMap::new(),
4184            max_concurrent_workflow_tasks: 10,
4185            max_concurrent_activity_tasks: 10,
4186            poll_timeout: Duration::from_secs(30),
4187            heartbeat_interval: Duration::from_secs(60),
4188            retry_policy: WorkerRetryPolicy::default(),
4189            heartbeat_observer: None,
4190        }
4191    }
4192
4193    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4194        self.worker_id = worker_id.into();
4195        self
4196    }
4197
4198    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4199        self.poll_timeout = timeout;
4200        self
4201    }
4202
4203    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4204        self.heartbeat_interval = interval;
4205        self
4206    }
4207
4208    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
4209    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4210        self.retry_policy = policy;
4211        self
4212    }
4213
4214    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4215    where
4216        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4217    {
4218        self.heartbeat_observer = Some(Arc::new(observer));
4219        self
4220    }
4221
4222    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4223        self.max_concurrent_workflow_tasks = count.max(1);
4224        self
4225    }
4226
4227    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4228        self.max_concurrent_activity_tasks = count.max(1);
4229        self
4230    }
4231
4232    /// Register a workflow handler.
4233    ///
4234    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
4235    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
4236    /// while acquiring or decoding a worker task remain worker-operation
4237    /// failures and do not get converted into workflow outcomes.
4238    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4239    where
4240        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4241        Fut: Future<Output = Result<Value>> + Send + 'static,
4242    {
4243        let handler = Arc::new(handler);
4244        self.workflows.insert(
4245            workflow_type.into(),
4246            RegisteredWorkflow {
4247                execute: Arc::new(move |ctx, input| {
4248                    let handler = Arc::clone(&handler);
4249                    Box::pin(async move {
4250                        let result = handler(ctx, input.into_json()?).await?;
4251                        AvroValue::from_serialize(&result)
4252                    })
4253                }),
4254                replay: None,
4255                state_type: None,
4256            },
4257        );
4258    }
4259
4260    /// Register a workflow on the lossless fixed Avro Value surface.
4261    pub fn register_workflow_avro_value<F, Fut>(
4262        &mut self,
4263        workflow_type: impl Into<String>,
4264        handler: F,
4265    ) where
4266        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4267        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4268    {
4269        self.workflows.insert(
4270            workflow_type.into(),
4271            RegisteredWorkflow {
4272                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4273                replay: None,
4274                state_type: None,
4275            },
4276        );
4277    }
4278
4279    /// Register a workflow whose typed instance state can be reconstructed for queries.
4280    ///
4281    /// `state_factory` creates a fresh instance for every normal workflow task and
4282    /// query replay. The workflow handler is the single source of truth for state
4283    /// transitions: it updates [`WorkflowInstance`] after activities and signals
4284    /// resolve. Query replay runs this same handler over committed history and
4285    /// discards any commands it would emit.
4286    pub fn register_replayed_workflow<S, Factory, F, Fut>(
4287        &mut self,
4288        workflow_type: impl Into<String>,
4289        state_factory: Factory,
4290        handler: F,
4291    ) where
4292        S: Clone + Send + Sync + 'static,
4293        Factory: Fn() -> S + Send + Sync + 'static,
4294        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4295        Fut: Future<Output = Result<Value>> + Send + 'static,
4296    {
4297        let state_factory = Arc::new(state_factory);
4298        let handler = Arc::new(handler);
4299
4300        let execute_factory = Arc::clone(&state_factory);
4301        let execute_handler = Arc::clone(&handler);
4302        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4303            let state = WorkflowInstance::new(execute_factory());
4304            let handler = Arc::clone(&execute_handler);
4305            Box::pin(async move {
4306                let result = handler(ctx, input.into_json()?, state).await?;
4307                AvroValue::from_serialize(&result)
4308            }) as WorkflowFuture
4309        });
4310
4311        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4312            let state = WorkflowInstance::new(state_factory());
4313            let snapshot_state = state.clone();
4314            let snapshot: WorkflowStateSnapshot =
4315                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4316            let replay_handler = Arc::clone(&handler);
4317            let future = async move {
4318                let result = replay_handler(ctx, input.into_json()?, state).await?;
4319                AvroValue::from_serialize(&result)
4320            };
4321            ReplayedWorkflowInvocation {
4322                future: Box::pin(future),
4323                snapshot,
4324            }
4325        });
4326
4327        self.workflows.insert(
4328            workflow_type.into(),
4329            RegisteredWorkflow {
4330                execute,
4331                replay: Some(replay),
4332                state_type: Some(TypeId::of::<S>()),
4333            },
4334        );
4335    }
4336
4337    /// Register a replayable workflow on the lossless fixed Avro Value surface.
4338    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4339        &mut self,
4340        workflow_type: impl Into<String>,
4341        state_factory: Factory,
4342        handler: F,
4343    ) where
4344        S: Clone + Send + Sync + 'static,
4345        Factory: Fn() -> S + Send + Sync + 'static,
4346        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4347        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4348    {
4349        let state_factory = Arc::new(state_factory);
4350        let handler = Arc::new(handler);
4351
4352        let execute_factory = Arc::clone(&state_factory);
4353        let execute_handler = Arc::clone(&handler);
4354        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4355            let state = WorkflowInstance::new(execute_factory());
4356            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4357        });
4358
4359        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4360            let state = WorkflowInstance::new(state_factory());
4361            let snapshot_state = state.clone();
4362            let snapshot: WorkflowStateSnapshot =
4363                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4364            ReplayedWorkflowInvocation {
4365                future: Box::pin(handler(ctx, input, state)),
4366                snapshot,
4367            }
4368        });
4369
4370        self.workflows.insert(
4371            workflow_type.into(),
4372            RegisteredWorkflow {
4373                execute,
4374                replay: Some(replay),
4375                state_type: Some(TypeId::of::<S>()),
4376            },
4377        );
4378    }
4379
4380    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4381    where
4382        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4383        Fut: Future<Output = Result<Value>> + Send + 'static,
4384    {
4385        let handler = Arc::new(handler);
4386        self.activities.insert(
4387            activity_type.into(),
4388            Arc::new(move |ctx, args| {
4389                let handler = Arc::clone(&handler);
4390                Box::pin(async move {
4391                    let result = handler(ctx, args.into_json()?).await?;
4392                    AvroValue::from_serialize(&result)
4393                })
4394            }),
4395        );
4396    }
4397
4398    /// Register an activity on the lossless fixed Avro Value surface.
4399    pub fn register_activity_avro_value<F, Fut>(
4400        &mut self,
4401        activity_type: impl Into<String>,
4402        handler: F,
4403    ) where
4404        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4405        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4406    {
4407        self.activities.insert(
4408            activity_type.into(),
4409            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4410        );
4411    }
4412
4413    /// Register a named, read-only query handler for a workflow type.
4414    ///
4415    /// The workflow type must also be registered with [`Worker::register_workflow`]
4416    /// before the worker runs. The handler receives only an immutable committed
4417    /// state snapshot and normalized query arguments.
4418    pub fn register_query<F, Fut>(
4419        &mut self,
4420        workflow_type: impl Into<String>,
4421        query_name: impl Into<String>,
4422        handler: F,
4423    ) where
4424        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4425        Fut: Future<Output = Result<Value>> + Send + 'static,
4426    {
4427        let handler = Arc::new(handler);
4428        self.queries
4429            .entry(workflow_type.into())
4430            .or_default()
4431            .insert(
4432                query_name.into(),
4433                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4434                    let handler = Arc::clone(&handler);
4435                    Box::pin(async move {
4436                        let result = handler(ctx, args.into_json()?).await?;
4437                        AvroValue::from_serialize(&result)
4438                    })
4439                })),
4440            );
4441    }
4442
4443    /// Register a query handler on the lossless fixed Avro Value surface.
4444    pub fn register_query_avro_value<F, Fut>(
4445        &mut self,
4446        workflow_type: impl Into<String>,
4447        query_name: impl Into<String>,
4448        handler: F,
4449    ) where
4450        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4451        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4452    {
4453        self.queries
4454            .entry(workflow_type.into())
4455            .or_default()
4456            .insert(
4457                query_name.into(),
4458                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4459            );
4460    }
4461
4462    /// Register a named query against deterministically replayed instance state.
4463    ///
4464    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
4465    /// same state type `S`. The handler receives an immutable, detached state
4466    /// clone, so successful and failed queries cannot affect workflow execution
4467    /// or the state reconstructed by a later query.
4468    pub fn register_replayed_query<S, F, Fut>(
4469        &mut self,
4470        workflow_type: impl Into<String>,
4471        query_name: impl Into<String>,
4472        handler: F,
4473    ) where
4474        S: Clone + Send + Sync + 'static,
4475        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4476        Fut: Future<Output = Result<Value>> + Send + 'static,
4477    {
4478        let handler = Arc::new(handler);
4479        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4480            let state = state.downcast::<S>().map_err(|_| {
4481                "registered query state type does not match the replayed workflow state".to_string()
4482            })?;
4483            let handler = Arc::clone(&handler);
4484            Ok(Box::pin(async move {
4485                let result = handler(ctx, state, args.into_json()?).await?;
4486                AvroValue::from_serialize(&result)
4487            }))
4488        });
4489
4490        self.queries
4491            .entry(workflow_type.into())
4492            .or_default()
4493            .insert(
4494                query_name.into(),
4495                RegisteredQuery::Replayed {
4496                    state_type: TypeId::of::<S>(),
4497                    handler: erased_handler,
4498                },
4499            );
4500    }
4501
4502    /// Register a replayed-state query on the lossless fixed Avro Value surface.
4503    pub fn register_replayed_query_avro_value<S, F, Fut>(
4504        &mut self,
4505        workflow_type: impl Into<String>,
4506        query_name: impl Into<String>,
4507        handler: F,
4508    ) where
4509        S: Clone + Send + Sync + 'static,
4510        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4511        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4512    {
4513        let handler = Arc::new(handler);
4514        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4515            let state = state.downcast::<S>().map_err(|_| {
4516                "registered query state type does not match the replayed workflow state".to_string()
4517            })?;
4518            Ok(Box::pin(handler(ctx, state, args)))
4519        });
4520
4521        self.queries
4522            .entry(workflow_type.into())
4523            .or_default()
4524            .insert(
4525                query_name.into(),
4526                RegisteredQuery::Replayed {
4527                    state_type: TypeId::of::<S>(),
4528                    handler: erased_handler,
4529                },
4530            );
4531    }
4532
4533    /// Register a synchronous workflow update handler.
4534    pub fn register_update<F, Fut>(
4535        &mut self,
4536        workflow_type: impl Into<String>,
4537        update_name: impl Into<String>,
4538        handler: F,
4539    ) where
4540        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4541        Fut: Future<Output = Result<Value>> + Send + 'static,
4542    {
4543        let handler = Arc::new(handler);
4544        self.updates
4545            .entry(workflow_type.into())
4546            .or_default()
4547            .insert(
4548                update_name.into(),
4549                Arc::new(move |ctx, args| {
4550                    let handler = Arc::clone(&handler);
4551                    Box::pin(async move {
4552                        let result = handler(ctx, args.into_json()?).await?;
4553                        AvroValue::from_serialize(&result)
4554                    })
4555                }),
4556            );
4557    }
4558
4559    /// Register an update handler on the lossless fixed Avro Value surface.
4560    pub fn register_update_avro_value<F, Fut>(
4561        &mut self,
4562        workflow_type: impl Into<String>,
4563        update_name: impl Into<String>,
4564        handler: F,
4565    ) where
4566        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4567        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4568    {
4569        self.updates
4570            .entry(workflow_type.into())
4571            .or_default()
4572            .insert(
4573                update_name.into(),
4574                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4575            );
4576    }
4577
4578    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4579        let mut command_contracts = serde_json::Map::new();
4580        for workflow_type in self.workflows.keys() {
4581            let mut queries = self
4582                .queries
4583                .get(workflow_type)
4584                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4585                .unwrap_or_default();
4586            queries.sort();
4587            let mut updates = self
4588                .updates
4589                .get(workflow_type)
4590                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4591                .unwrap_or_default();
4592            updates.sort();
4593            if !queries.is_empty() || !updates.is_empty() {
4594                command_contracts.insert(
4595                    workflow_type.clone(),
4596                    json!({
4597                        "queries": queries,
4598                        "updates": updates,
4599                        "update_validators": [],
4600                    }),
4601                );
4602            }
4603        }
4604
4605        self.client
4606            .register_worker_with_command_contracts(
4607                &self.worker_id,
4608                &self.task_queue,
4609                self.workflows.keys().cloned().collect(),
4610                self.activities.keys().cloned().collect(),
4611                self.max_concurrent_workflow_tasks,
4612                self.max_concurrent_activity_tasks,
4613                [
4614                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4615                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4616                ]
4617                .into_iter()
4618                .flatten()
4619                .collect(),
4620                Value::Object(command_contracts),
4621            )
4622            .await
4623    }
4624
4625    /// Run until shutdown or a terminal worker error occurs.
4626    ///
4627    /// Empty long-poll expirations do not stop the worker. Retryable poll and
4628    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
4629    /// authentication, protocol, and other non-retryable failures are returned.
4630    pub async fn run(&self) -> Result<()> {
4631        self.run_until(std::future::pending::<()>()).await
4632    }
4633
4634    /// Run until `shutdown` resolves or a terminal worker error occurs.
4635    ///
4636    /// This has the same liveness and terminal-error contract as [`Worker::run`].
4637    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4638    where
4639        F: Future<Output = ()>,
4640    {
4641        let registration = self.register().await?;
4642        if !registration.registered {
4643            return Err(Error::WorkerLoop(format!(
4644                "worker registration for {:?} was not accepted",
4645                self.worker_id
4646            )));
4647        }
4648        let registered_worker_id = registration.worker_id.clone();
4649        let primary = self.run_registered_until(shutdown, registration).await;
4650        let deregistration = self
4651            .client
4652            .deregister_worker_registration(&registered_worker_id)
4653            .await;
4654
4655        match (primary, deregistration) {
4656            (Ok(()), Ok(_)) => Ok(()),
4657            (Ok(()), Err(deregistration)) => Err(deregistration),
4658            (Err(primary), Ok(_)) => Err(primary),
4659            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4660                primary: Box::new(primary),
4661                deregistration: Box::new(deregistration),
4662            }),
4663        }
4664    }
4665
4666    async fn run_registered_until<F>(
4667        &self,
4668        shutdown: F,
4669        registration: RegisterWorkerResponse,
4670    ) -> Result<()>
4671    where
4672        F: Future<Output = ()>,
4673    {
4674        let heartbeat_interval = Duration::from_secs(
4675            registration
4676                .heartbeat_interval_seconds
4677                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4678        );
4679        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
4680        // from the completion of the preceding attempt, including its bounded
4681        // retries. A fixed-epoch interval can leave an already-due tick queued
4682        // while an acknowledgement is slow, producing a catch-up heartbeat as
4683        // soon as that request completes.
4684        let heartbeat = tokio::time::sleep(Duration::ZERO);
4685        tokio::pin!(heartbeat);
4686        tokio::pin!(shutdown);
4687        let stop = Arc::new(AtomicBool::new(false));
4688        // Poll responses may already have leased server-side work by the time
4689        // they become ready, so each poller owns its responses through
4690        // completion or failure instead of racing raw polls in this select.
4691        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4692            let worker = self.clone();
4693            let stop = Arc::clone(&stop);
4694            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4695        });
4696        let mut activity_poller = (!self.activities.is_empty()).then(|| {
4697            let worker = self.clone();
4698            let stop = Arc::clone(&stop);
4699            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4700        });
4701        let mut query_poller = (!self.queries.is_empty()).then(|| {
4702            let worker = self.clone();
4703            let stop = Arc::clone(&stop);
4704            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4705        });
4706
4707        loop {
4708            tokio::select! {
4709                _ = &mut shutdown => {
4710                    stop.store(true, Ordering::SeqCst);
4711                    break;
4712                }
4713                _ = &mut heartbeat => {
4714                    let result = self.retry_worker_operation(|| {
4715                        self.client.heartbeat_worker(
4716                            &self.worker_id,
4717                            self.max_concurrent_workflow_tasks,
4718                            self.max_concurrent_activity_tasks,
4719                        )
4720                    }).await;
4721                    heartbeat
4722                        .as_mut()
4723                        .reset(tokio::time::Instant::now() + heartbeat_interval);
4724                    match result {
4725                        Ok(acknowledgement) => {
4726                            if let Some(observer) = &self.heartbeat_observer {
4727                                observer(&WorkerHeartbeatObservation {
4728                                    worker_id: self.worker_id.clone(),
4729                                    task_queue: self.task_queue.clone(),
4730                                    acknowledged_at_unix_millis: SystemTime::now()
4731                                        .duration_since(UNIX_EPOCH)
4732                                        .unwrap_or_default()
4733                                        .as_millis()
4734                                        .min(u64::MAX as u128)
4735                                        as u64,
4736                                    acknowledgement,
4737                                });
4738                            }
4739                        }
4740                        Err(error) => {
4741                            stop.store(true, Ordering::SeqCst);
4742                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4743                            return Err(error);
4744                        }
4745                    }
4746                }
4747                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4748                    workflow_poller = None;
4749                    let stopped_by_server = stop.load(Ordering::SeqCst);
4750                    stop.store(true, Ordering::SeqCst);
4751                    let poller_result = optional_poller_result("workflow", result);
4752                    let join_result =
4753                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4754                    poller_result?;
4755                    join_result?;
4756                    if stopped_by_server {
4757                        return Ok(());
4758                    }
4759                    return Err(Error::WorkerLoop(
4760                        "workflow poller stopped unexpectedly".to_string(),
4761                    ));
4762                }
4763                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4764                    activity_poller = None;
4765                    let stopped_by_server = stop.load(Ordering::SeqCst);
4766                    stop.store(true, Ordering::SeqCst);
4767                    let poller_result = optional_poller_result("activity", result);
4768                    let join_result =
4769                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4770                    poller_result?;
4771                    join_result?;
4772                    if stopped_by_server {
4773                        return Ok(());
4774                    }
4775                    return Err(Error::WorkerLoop(
4776                        "activity poller stopped unexpectedly".to_string(),
4777                    ));
4778                }
4779                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4780                    query_poller = None;
4781                    let stopped_by_server = stop.load(Ordering::SeqCst);
4782                    stop.store(true, Ordering::SeqCst);
4783                    let poller_result = optional_poller_result("query", result);
4784                    let join_result =
4785                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4786                    poller_result?;
4787                    join_result?;
4788                    if stopped_by_server {
4789                        return Ok(());
4790                    }
4791                    return Err(Error::WorkerLoop(
4792                        "query poller stopped unexpectedly".to_string(),
4793                    ));
4794                }
4795            }
4796        }
4797
4798        join_pollers(
4799            workflow_poller.take(),
4800            activity_poller.take(),
4801            query_poller.take(),
4802        )
4803        .await
4804    }
4805
4806    /// Poll and settle at most one task from each enabled task family.
4807    ///
4808    /// A workflow may reach its server-enforced run deadline while this worker
4809    /// holds a task. When the completion endpoint authoritatively rejects that
4810    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
4811    /// terminal `run_status=failed`, the workflow tick is considered settled:
4812    /// the late command was not recorded and cannot replace the terminal run.
4813    /// Every other completion rejection remains an error. This worker-level
4814    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
4815    /// only bounds how long a client waits for a result.
4816    ///
4817    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
4818    /// the original [`Error::Http`] status and response body.
4819    pub async fn run_once(&self) -> Result<usize> {
4820        let mut handled = 0;
4821        match self.poll_workflow_once().await? {
4822            ManagedPollOutcome::Handled => handled += 1,
4823            ManagedPollOutcome::Stop => return Ok(handled),
4824            ManagedPollOutcome::Idle => {}
4825        }
4826        match self.poll_activity_once().await? {
4827            ManagedPollOutcome::Handled => handled += 1,
4828            ManagedPollOutcome::Stop => return Ok(handled),
4829            ManagedPollOutcome::Idle => {}
4830        }
4831        if !self.queries.is_empty() {
4832            match self.poll_query_once().await? {
4833                ManagedPollOutcome::Handled => handled += 1,
4834                ManagedPollOutcome::Stop => return Ok(handled),
4835                ManagedPollOutcome::Idle => {}
4836            }
4837        }
4838        Ok(handled)
4839    }
4840
4841    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4842        let poll_request_id = unique_request_id("rust-workflow-poll");
4843        let response = self
4844            .retry_worker_operation(|| {
4845                self.client.poll_workflow_task_response_with_request_id(
4846                    &self.worker_id,
4847                    &self.task_queue,
4848                    self.poll_timeout,
4849                    &poll_request_id,
4850                    0,
4851                )
4852            })
4853            .await?;
4854        if response.outcome().should_stop() {
4855            return Ok(ManagedPollOutcome::Stop);
4856        }
4857        let Some(task) = response.task else {
4858            return Ok(ManagedPollOutcome::Idle);
4859        };
4860
4861        let task_id = task.task_id.clone();
4862        let attempt = task.workflow_task_attempt;
4863        let run_id = task.run_id.clone();
4864        let lease_owner = task
4865            .lease_owner
4866            .clone()
4867            .unwrap_or_else(|| self.worker_id.clone());
4868
4869        match self.execute_workflow_task(task) {
4870            Ok(commands) if commands.is_empty() => {
4871                // A replay can consume a recorded pending durable command
4872                // without producing a new command. The standalone protocol
4873                // acknowledges that state through the typed waiting outcome;
4874                // an empty completion is rejected by servers that require at
4875                // least one executable command.
4876                self.client
4877                    .fail_workflow_task_with_type(
4878                        &task_id,
4879                        &lease_owner,
4880                        attempt,
4881                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4882                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4883                    )
4884                    .await?;
4885            }
4886            Ok(commands) => {
4887                let completion = self
4888                    .client
4889                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4890                    .await;
4891                if let Err(error) = completion {
4892                    if !workflow_task_completion_is_terminal_timeout(
4893                        &error,
4894                        &task_id,
4895                        attempt,
4896                        run_id.as_deref(),
4897                    ) {
4898                        return Err(error);
4899                    }
4900                }
4901            }
4902            Err(error) => {
4903                self.client
4904                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4905                    .await?;
4906            }
4907        }
4908
4909        Ok(ManagedPollOutcome::Handled)
4910    }
4911
4912    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4913        while !stop.load(Ordering::SeqCst) {
4914            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4915                stop.store(true, Ordering::SeqCst);
4916                break;
4917            }
4918        }
4919
4920        Ok(())
4921    }
4922
4923    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4924        let poll_request_id = unique_request_id("rust-activity-poll");
4925        let response = self
4926            .retry_worker_operation(|| {
4927                self.client.poll_activity_task_response_with_request_id(
4928                    &self.worker_id,
4929                    &self.task_queue,
4930                    self.poll_timeout,
4931                    &poll_request_id,
4932                    0,
4933                )
4934            })
4935            .await?;
4936        if response.outcome().should_stop() {
4937            return Ok(ManagedPollOutcome::Stop);
4938        }
4939        let Some(task) = response.task else {
4940            return Ok(ManagedPollOutcome::Idle);
4941        };
4942
4943        let task_id = task.task_id.clone();
4944        let attempt_id = task
4945            .activity_attempt_id
4946            .clone()
4947            .or(task.attempt_id.clone())
4948            .unwrap_or_default();
4949        let lease_owner = task
4950            .lease_owner
4951            .clone()
4952            .unwrap_or_else(|| self.worker_id.clone());
4953        let codec = task.payload_codec.clone();
4954        let result = self.execute_activity_task(task).await;
4955        match result {
4956            Ok(value) => {
4957                let completion = self
4958                    .client
4959                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4960                    .await;
4961                if let Err(error) = completion {
4962                    if !activity_task_rejection_is_final(&error) {
4963                        return Err(error);
4964                    }
4965                }
4966            }
4967            Err(error) => {
4968                let failure = self
4969                    .client
4970                    .fail_activity_task(
4971                        &task_id,
4972                        &attempt_id,
4973                        &lease_owner,
4974                        error.to_string(),
4975                        false,
4976                    )
4977                    .await;
4978                if let Err(error) = failure {
4979                    if !activity_task_rejection_is_final(&error) {
4980                        return Err(error);
4981                    }
4982                }
4983            }
4984        }
4985
4986        Ok(ManagedPollOutcome::Handled)
4987    }
4988
4989    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4990        while !stop.load(Ordering::SeqCst) {
4991            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4992                stop.store(true, Ordering::SeqCst);
4993                break;
4994            }
4995        }
4996
4997        Ok(())
4998    }
4999
5000    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
5001        let poll_request_id = unique_request_id("rust-query-poll");
5002        let response = self
5003            .retry_worker_operation(|| {
5004                self.client.poll_query_task_response_with_request_id(
5005                    &self.worker_id,
5006                    &self.task_queue,
5007                    self.poll_timeout,
5008                    &poll_request_id,
5009                    0,
5010                )
5011            })
5012            .await?;
5013        if response.outcome().should_stop() {
5014            return Ok(ManagedPollOutcome::Stop);
5015        }
5016        let Some(task) = response.task else {
5017            return Ok(ManagedPollOutcome::Idle);
5018        };
5019
5020        let query_task_id = task.query_task_id.clone();
5021        let attempt = task.query_task_attempt;
5022        let lease_owner = task
5023            .lease_owner
5024            .clone()
5025            .unwrap_or_else(|| self.worker_id.clone());
5026        let codec = task.payload_codec.clone();
5027
5028        match self.execute_query_task(task).await {
5029            Ok(value) => {
5030                let result_envelope = match encode_typed_envelope(&value, &codec) {
5031                    Ok(result_envelope) => result_envelope,
5032                    Err(error) => {
5033                        let failure = self
5034                            .client
5035                            .fail_query_task(
5036                                &query_task_id,
5037                                &lease_owner,
5038                                attempt,
5039                                error.to_string(),
5040                                "query_result_encode_failed",
5041                                "QueryResultEncodeFailed",
5042                            )
5043                            .await;
5044                        if let Err(error) = failure {
5045                            if !query_task_rejection_is_final(&error) {
5046                                return Err(error);
5047                            }
5048                        }
5049                        return Ok(ManagedPollOutcome::Handled);
5050                    }
5051                };
5052
5053                if let Err(error) = self
5054                    .client
5055                    .complete_query_task_with_envelope(
5056                        &query_task_id,
5057                        &lease_owner,
5058                        attempt,
5059                        value.clone().into_json()?,
5060                        result_envelope,
5061                    )
5062                    .await
5063                {
5064                    if !query_task_rejection_is_final(&error) {
5065                        return Err(error);
5066                    }
5067                }
5068            }
5069            Err(failure) => {
5070                let result = self
5071                    .client
5072                    .fail_query_task(
5073                        &query_task_id,
5074                        &lease_owner,
5075                        attempt,
5076                        failure.message,
5077                        failure.reason,
5078                        failure.failure_type,
5079                    )
5080                    .await;
5081                if let Err(error) = result {
5082                    if !query_task_rejection_is_final(&error) {
5083                        return Err(error);
5084                    }
5085                }
5086            }
5087        }
5088
5089        Ok(ManagedPollOutcome::Handled)
5090    }
5091
5092    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5093        while !stop.load(Ordering::SeqCst) {
5094            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5095                stop.store(true, Ordering::SeqCst);
5096                break;
5097            }
5098        }
5099
5100        Ok(())
5101    }
5102
5103    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5104    where
5105        F: FnMut() -> Fut,
5106        Fut: Future<Output = Result<T>>,
5107    {
5108        let mut retries = 0;
5109
5110        loop {
5111            match operation().await {
5112                Err(error)
5113                    if worker_operation_is_retryable(&error)
5114                        && retries < self.retry_policy.max_retries =>
5115                {
5116                    retries += 1;
5117                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5118                }
5119                result => return result,
5120            }
5121        }
5122    }
5123
5124    async fn execute_query_task(
5125        &self,
5126        mut task: QueryTask,
5127    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5128        validate_query_task_payloads(&task).map_err(|error| {
5129            QueryTaskExecutionFailure::new(
5130                "query_payload_decode_failed",
5131                error.to_string(),
5132                "QueryPayloadDecodeFailed",
5133            )
5134        })?;
5135
5136        if !self.workflows.contains_key(&task.workflow_type) {
5137            return Err(QueryTaskExecutionFailure::new(
5138                "query_workflow_type_not_registered",
5139                format!("no workflow registered for type {:?}", task.workflow_type),
5140                "WorkflowTypeNotRegistered",
5141            ));
5142        }
5143
5144        let Some(handlers) = self.queries.get(&task.workflow_type) else {
5145            return Err(QueryTaskExecutionFailure::new(
5146                "query_handler_unavailable",
5147                format!(
5148                    "query handlers are unavailable for workflow type {:?}",
5149                    task.workflow_type
5150                ),
5151                "QueryHandlerUnavailable",
5152            ));
5153        };
5154        let Some(query) = handlers.get(&task.query_name) else {
5155            return Err(QueryTaskExecutionFailure::new(
5156                "rejected_unknown_query",
5157                format!("unknown query {:?}", task.query_name),
5158                "QueryFailed",
5159            ));
5160        };
5161
5162        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5163            .map_err(|error| {
5164            QueryTaskExecutionFailure::new(
5165                "query_payload_decode_failed",
5166                format!("cannot decode query arguments: {error}"),
5167                "QueryPayloadDecodeFailed",
5168            )
5169        })?;
5170        let workflow_input_typed =
5171            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5172                .map_err(|error| {
5173                    QueryTaskExecutionFailure::new(
5174                        "query_workflow_state_unavailable",
5175                        format!("cannot decode workflow start input: {error}"),
5176                        "QueryWorkflowStateUnavailable",
5177                    )
5178                })?;
5179        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5180            QueryTaskExecutionFailure::new(
5181                "query_workflow_state_unavailable",
5182                format!("cannot project workflow start input: {error}"),
5183                "QueryWorkflowStateUnavailable",
5184            )
5185        })?;
5186        hydrate_query_history_from_export(&mut task).map_err(|error| {
5187            QueryTaskExecutionFailure::new(
5188                "query_workflow_state_unavailable",
5189                format!("cannot restore query history snapshot: {error}"),
5190                "QueryWorkflowStateUnavailable",
5191            )
5192        })?;
5193        enrich_query_history_from_export(&mut task).map_err(|error| {
5194            QueryTaskExecutionFailure::new(
5195                "query_workflow_state_unavailable",
5196                format!("cannot restore compact query history payloads: {error}"),
5197                "QueryWorkflowStateUnavailable",
5198            )
5199        })?;
5200        let signal_events = query_signal_events(&task).map_err(|error| {
5201            QueryTaskExecutionFailure::new(
5202                "query_workflow_state_unavailable",
5203                format!("cannot decode committed workflow signals: {error}"),
5204                "QueryWorkflowStateUnavailable",
5205            )
5206        })?;
5207        let history_events = Arc::new(std::mem::take(&mut task.history_events));
5208        let context = QueryContext {
5209            workflow_id: task.workflow_id,
5210            run_id: task.run_id,
5211            workflow_type: task.workflow_type.clone(),
5212            run_status: task.run_status,
5213            workflow_input,
5214            workflow_input_avro_value: workflow_input_typed.clone(),
5215            history_events: Arc::clone(&history_events),
5216            signal_events: Arc::new(signal_events),
5217        };
5218
5219        let future = match query {
5220            RegisteredQuery::Snapshot(handler) => handler(context, args),
5221            RegisteredQuery::Replayed {
5222                state_type,
5223                handler,
5224            } => {
5225                let workflow = self
5226                    .workflows
5227                    .get(&task.workflow_type)
5228                    .expect("workflow registration was checked above");
5229                if workflow.state_type != Some(*state_type) {
5230                    return Err(QueryTaskExecutionFailure::new(
5231                        "query_workflow_state_unavailable",
5232                        "replayed query state type does not match its workflow registration",
5233                        "QueryWorkflowStateUnavailable",
5234                    ));
5235                }
5236                let replay = workflow.replay.as_ref().ok_or_else(|| {
5237                    QueryTaskExecutionFailure::new(
5238                        "query_workflow_state_unavailable",
5239                        format!(
5240                            "workflow type {:?} is not registered for instance-state replay",
5241                            task.workflow_type
5242                        ),
5243                        "QueryWorkflowStateUnavailable",
5244                    )
5245                })?;
5246                let workflow_state = Arc::new(Mutex::new(
5247                    WorkflowState::new_with_identity(
5248                        history_events.as_ref().clone(),
5249                        context.workflow_id.clone(),
5250                        context.run_id.clone(),
5251                        self.task_queue.clone(),
5252                        task.payload_codec,
5253                        None,
5254                    )
5255                    .map_err(|error| {
5256                        QueryTaskExecutionFailure::new(
5257                            "query_workflow_state_unavailable",
5258                            format!("workflow replay failed before query: {error}"),
5259                            "QueryWorkflowStateUnavailable",
5260                        )
5261                    })?,
5262                ));
5263                let workflow_context = WorkflowContext {
5264                    state: workflow_state,
5265                };
5266                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5267                let mut cx = TaskContext::from_waker(noop_waker_ref());
5268                match invocation.future.as_mut().poll(&mut cx) {
5269                    Poll::Ready(Ok(_)) => {
5270                        workflow_context
5271                            .ensure_history_consumed()
5272                            .map_err(|error| {
5273                                QueryTaskExecutionFailure::new(
5274                                    "query_workflow_state_unavailable",
5275                                    format!("workflow replay failed before query: {error}"),
5276                                    "QueryWorkflowStateUnavailable",
5277                                )
5278                            })?;
5279                    }
5280                    Poll::Ready(Err(error)) => {
5281                        return Err(QueryTaskExecutionFailure::new(
5282                            "query_workflow_state_unavailable",
5283                            format!("workflow replay failed before query: {error}"),
5284                            "QueryWorkflowStateUnavailable",
5285                        ));
5286                    }
5287                    Poll::Pending => {
5288                        let commands = workflow_context.take_commands().map_err(|error| {
5289                            QueryTaskExecutionFailure::new(
5290                                "query_workflow_state_unavailable",
5291                                format!("workflow replay failed before query: {error}"),
5292                                "QueryWorkflowStateUnavailable",
5293                            )
5294                        })?;
5295                        if commands.is_empty()
5296                            && !workflow_context
5297                                .matched_recorded_pending()
5298                                .map_err(|error| {
5299                                    QueryTaskExecutionFailure::new(
5300                                        "query_workflow_state_unavailable",
5301                                        format!("workflow replay failed before query: {error}"),
5302                                        "QueryWorkflowStateUnavailable",
5303                                    )
5304                                })?
5305                        {
5306                            return Err(QueryTaskExecutionFailure::new(
5307                                "query_workflow_state_unavailable",
5308                                "workflow replay yielded without a durable command",
5309                                "QueryWorkflowStateUnavailable",
5310                            ));
5311                        }
5312                    }
5313                }
5314                let state = (invocation.snapshot)().map_err(|error| {
5315                    QueryTaskExecutionFailure::new(
5316                        "query_workflow_state_unavailable",
5317                        format!("cannot snapshot replayed workflow state: {error}"),
5318                        "QueryWorkflowStateUnavailable",
5319                    )
5320                })?;
5321                handler(context, state, args).map_err(|message| {
5322                    QueryTaskExecutionFailure::new(
5323                        "query_workflow_state_unavailable",
5324                        message,
5325                        "QueryWorkflowStateUnavailable",
5326                    )
5327                })?
5328            }
5329        };
5330
5331        future.await.map_err(|error| {
5332            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5333        })
5334    }
5335
5336    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5337        validate_workflow_task_payloads(&task)?;
5338
5339        if let Some(update_id) = task
5340            .workflow_update_id
5341            .as_deref()
5342            .filter(|update_id| !update_id.is_empty())
5343        {
5344            return self.execute_update_task(&task, update_id);
5345        }
5346
5347        let workflow = self
5348            .workflows
5349            .get(&task.workflow_type)
5350            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5351        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5352        let resume_signal = decode_resume_signal(&task)?;
5353        let history_budget = WorkflowHistoryBudget {
5354            event_count: task
5355                .total_history_events
5356                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5357            size_bytes: task.history_size_bytes,
5358            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5359            pressure: task.history_budget_pressure.clone(),
5360        };
5361        let mut workflow_state = WorkflowState::new_with_identity(
5362            task.history_events,
5363            task.workflow_id,
5364            task.run_id,
5365            self.task_queue.clone(),
5366            task.payload_codec.clone(),
5367            resume_signal,
5368        )?;
5369        workflow_state.history_budget = history_budget;
5370        let state = Arc::new(Mutex::new(workflow_state));
5371        let ctx = WorkflowContext { state };
5372        let mut future = (workflow.execute)(ctx.clone(), input);
5373        let mut cx = TaskContext::from_waker(noop_waker_ref());
5374
5375        match future.as_mut().poll(&mut cx) {
5376            Poll::Ready(Ok(result)) => {
5377                ctx.ensure_history_consumed()?;
5378                let result = encode_typed_envelope(&result, &task.payload_codec)?;
5379                let mut commands = ctx.take_commands()?;
5380                commands.push(json!({
5381                    "type": "complete_workflow",
5382                    "result": result
5383                }));
5384                Ok(commands)
5385            }
5386            Poll::Ready(Err(error)) => {
5387                if let Error::ContinueAsNew(request) = error {
5388                    let mut commands = ctx.take_commands()?;
5389                    if let Some(command) = ctx.continue_as_new_command(request)? {
5390                        commands.push(command);
5391                    }
5392                    ctx.ensure_history_consumed()?;
5393                    return Ok(commands);
5394                }
5395                // A handler error must not hide a committed durable command that
5396                // upgraded workflow code no longer consumes.
5397                ctx.ensure_history_consumed()?;
5398                if workflow_task_integrity_error(&error) {
5399                    // Replay and protocol failures describe the workflow-task
5400                    // decision itself. Do not let commands queued earlier in
5401                    // this uncommitted decision escape alongside a terminal
5402                    // workflow failure.
5403                    return Err(error);
5404                }
5405                let mut commands = ctx.take_commands()?;
5406                commands.push(workflow_failure_command(&error));
5407                Ok(commands)
5408            }
5409            Poll::Pending => {
5410                let commands = ctx.take_commands()?;
5411                if commands.is_empty() && !ctx.matched_recorded_pending()? {
5412                    Err(Error::WorkflowYieldedWithoutCommand)
5413                } else {
5414                    Ok(commands)
5415                }
5416            }
5417        }
5418    }
5419
5420    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5421        if !self.workflows.contains_key(&task.workflow_type) {
5422            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5423        }
5424
5425        let accepted = task.history_events.iter().rev().find_map(|event| {
5426            (event.event_type == "UpdateAccepted"
5427                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5428            .then_some(&event.payload)
5429        });
5430        let update_name = accepted
5431            .and_then(|payload| payload.get("update_name"))
5432            .and_then(Value::as_str)
5433            .or(task.update_name.as_deref())
5434            .unwrap_or_default();
5435        let Some(handler) = self
5436            .updates
5437            .get(&task.workflow_type)
5438            .and_then(|handlers| handlers.get(update_name))
5439        else {
5440            return Ok(vec![json!({
5441                "type": "fail_update",
5442                "update_id": update_id,
5443                "message": format!(
5444                    "no update handler is registered for {}.{update_name}",
5445                    task.workflow_type
5446                ),
5447                "exception_type": "UnknownUpdate",
5448                "non_retryable": true,
5449            })]);
5450        };
5451        let arguments = accepted
5452            .and_then(|payload| payload.get("arguments"))
5453            .or(task.arguments.as_ref());
5454        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5455        let context = QueryContext {
5456            workflow_id: task.workflow_id.clone(),
5457            run_id: task.run_id.clone(),
5458            workflow_type: task.workflow_type.clone(),
5459            run_status: Some("running".to_string()),
5460            workflow_input: Value::Null,
5461            workflow_input_avro_value: AvroValue::Null,
5462            history_events: Arc::new(task.history_events.clone()),
5463            signal_events: Arc::new(Vec::new()),
5464        };
5465        let mut future = handler(context, arguments);
5466        let mut cx = TaskContext::from_waker(noop_waker_ref());
5467
5468        match future.as_mut().poll(&mut cx) {
5469            Poll::Ready(Ok(result)) => Ok(vec![json!({
5470                "type": "complete_update",
5471                "update_id": update_id,
5472                "result": encode_typed_envelope(&result, &task.payload_codec)?,
5473            })]),
5474            Poll::Ready(Err(error)) => Ok(vec![json!({
5475                "type": "fail_update",
5476                "update_id": update_id,
5477                "message": error.to_string(),
5478                "exception_type": "UpdateFailed",
5479                "non_retryable": true,
5480            })]),
5481            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5482        }
5483    }
5484
5485    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5486        validate_activity_task_payloads(&task)?;
5487
5488        let handler = self
5489            .activities
5490            .get(&task.activity_type)
5491            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5492        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5493        let attempt_id = task
5494            .activity_attempt_id
5495            .clone()
5496            .or(task.attempt_id.clone())
5497            .unwrap_or_default();
5498        let lease_owner = task
5499            .lease_owner
5500            .clone()
5501            .unwrap_or_else(|| self.worker_id.clone());
5502        let ctx = ActivityContext {
5503            client: self.client.clone(),
5504            task_id: task.task_id,
5505            activity_attempt_id: attempt_id,
5506            lease_owner,
5507            activity_type: task.activity_type,
5508            attempt_number: task.attempt_number,
5509            task_queue: self.task_queue.clone(),
5510            worker_id: self.worker_id.clone(),
5511        };
5512
5513        handler(ctx, args).await
5514    }
5515}
5516
5517fn poller_result(
5518    kind: &str,
5519    result: std::result::Result<Result<()>, tokio::task::JoinError>,
5520) -> Result<()> {
5521    match result {
5522        Ok(result) => result,
5523        Err(error) => Err(Error::WorkerLoop(format!(
5524            "{kind} poller join error: {error}"
5525        ))),
5526    }
5527}
5528
5529fn optional_poller_result(
5530    kind: &str,
5531    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5532) -> Result<()> {
5533    match result {
5534        Some(result) => poller_result(kind, result),
5535        None => Ok(()),
5536    }
5537}
5538
5539async fn join_pollers(
5540    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5541    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5542    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5543) -> Result<()> {
5544    let mut first_error = None;
5545
5546    if let Some(handle) = workflow_poller {
5547        if let Err(error) = poller_result("workflow", handle.await) {
5548            first_error.get_or_insert(error);
5549        }
5550    }
5551
5552    if let Some(handle) = activity_poller {
5553        if let Err(error) = poller_result("activity", handle.await) {
5554            first_error.get_or_insert(error);
5555        }
5556    }
5557
5558    if let Some(handle) = query_poller {
5559        if let Err(error) = poller_result("query", handle.await) {
5560            first_error.get_or_insert(error);
5561        }
5562    }
5563
5564    if let Some(error) = first_error {
5565        Err(error)
5566    } else {
5567        Ok(())
5568    }
5569}
5570
5571fn default_worker_id() -> String {
5572    let millis = SystemTime::now()
5573        .duration_since(UNIX_EPOCH)
5574        .unwrap_or_default()
5575        .as_millis();
5576    format!("rust-worker-{}-{millis}", std::process::id())
5577}
5578
5579fn percent_encode_path_segment(segment: &str) -> String {
5580    const HEX: &[u8; 16] = b"0123456789ABCDEF";
5581    let mut encoded = String::with_capacity(segment.len());
5582
5583    for byte in segment.bytes() {
5584        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5585            encoded.push(char::from(byte));
5586        } else {
5587            encoded.push('%');
5588            encoded.push(char::from(HEX[(byte >> 4) as usize]));
5589            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5590        }
5591    }
5592
5593    encoded
5594}
5595
5596fn unique_request_id(prefix: &str) -> String {
5597    let nanos = SystemTime::now()
5598        .duration_since(UNIX_EPOCH)
5599        .unwrap_or_default()
5600        .as_nanos();
5601    format!("{prefix}-{}-{nanos}", std::process::id())
5602}
5603
5604#[derive(Debug)]
5605struct QueryTaskExecutionFailure {
5606    reason: String,
5607    message: String,
5608    failure_type: String,
5609}
5610
5611impl QueryTaskExecutionFailure {
5612    fn new(
5613        reason: impl Into<String>,
5614        message: impl Into<String>,
5615        failure_type: impl Into<String>,
5616    ) -> Self {
5617        Self {
5618            reason: reason.into(),
5619            message: message.into(),
5620            failure_type: failure_type.into(),
5621        }
5622    }
5623}
5624
5625/// Typed local state owned by one deterministic workflow invocation.
5626///
5627/// Use [`WorkflowInstance::update`] for the same state transitions during
5628/// ordinary execution and replay. A replayed query receives a detached
5629/// immutable `Arc<S>` rather than this mutation-capable handle.
5630#[derive(Clone, Debug)]
5631pub struct WorkflowInstance<S> {
5632    state: Arc<Mutex<S>>,
5633}
5634
5635impl<S> WorkflowInstance<S> {
5636    fn new(state: S) -> Self {
5637        Self {
5638            state: Arc::new(Mutex::new(state)),
5639        }
5640    }
5641
5642    /// Read the current workflow-instance state without changing it.
5643    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5644        let state = self
5645            .state
5646            .lock()
5647            .map_err(|_| Error::WorkflowStatePoisoned)?;
5648        Ok(reader(&state))
5649    }
5650
5651    /// Apply one deterministic workflow-instance state transition.
5652    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5653        let mut state = self
5654            .state
5655            .lock()
5656            .map_err(|_| Error::WorkflowStatePoisoned)?;
5657        Ok(transition(&mut state))
5658    }
5659}
5660
5661impl<S: Clone> WorkflowInstance<S> {
5662    fn snapshot(&self) -> Result<S> {
5663        self.read(Clone::clone)
5664    }
5665}
5666
5667#[derive(Clone, Debug)]
5668pub struct WorkflowContext {
5669    state: Arc<Mutex<WorkflowState>>,
5670}
5671
5672impl WorkflowContext {
5673    /// Identity of the parent workflow currently being replayed.
5674    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5675        let state = self
5676            .state
5677            .lock()
5678            .map_err(|_| Error::WorkflowStatePoisoned)?;
5679        Ok(WorkflowIdentity {
5680            workflow_id: state.workflow_id.clone(),
5681            run_id: state.run_id.clone(),
5682        })
5683    }
5684
5685    /// Return the server-published history budget for this workflow task.
5686    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5687        let state = self
5688            .state
5689            .lock()
5690            .map_err(|_| Error::WorkflowStatePoisoned)?;
5691        Ok(state.history_budget.clone())
5692    }
5693
5694    /// Continue this workflow instance as a fresh run with replacement arguments.
5695    ///
5696    /// Return this value directly from the workflow handler. The worker converts
5697    /// it to the terminal protocol command only after replay has consumed every
5698    /// recorded durable command.
5699    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5700        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5701    }
5702
5703    /// Continue as new with optional workflow-type and task-queue overrides.
5704    pub fn continue_as_new_with_options<T: Serialize>(
5705        &self,
5706        options: ContinueAsNewOptions,
5707        args: T,
5708    ) -> Result<Value> {
5709        options.validate()?;
5710        Err(Error::ContinueAsNew(ContinueAsNewRequest {
5711            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5712            options,
5713        }))
5714    }
5715
5716    pub fn activity<T: Serialize>(
5717        &self,
5718        activity_type: impl Into<String>,
5719        args: T,
5720    ) -> ActivityCall {
5721        self.activity_with_options(activity_type, ActivityOptions::new(), args)
5722    }
5723
5724    pub fn activity_on_queue<T, Q>(
5725        &self,
5726        activity_type: impl Into<String>,
5727        task_queue: Option<Q>,
5728        args: T,
5729    ) -> ActivityCall
5730    where
5731        T: Serialize,
5732        Q: Into<String>,
5733    {
5734        let mut options = ActivityOptions::new();
5735        options.task_queue = task_queue.map(Into::into);
5736        self.activity_with_options(activity_type, options, args)
5737    }
5738
5739    /// Schedule one durable activity with retry, routing, and timeout options.
5740    ///
5741    /// Options are validated before the command is emitted. Once the command is
5742    /// recorded, replay consumes the same activity lifecycle at this command
5743    /// position and never emits a duplicate schedule.
5744    ///
5745    /// ```no_run
5746    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
5747    /// # use std::time::Duration;
5748    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
5749    /// let result = ctx
5750    ///     .activity_with_options(
5751    ///         "charge-card",
5752    ///         ActivityOptions::new()
5753    ///             .task_queue("payments")
5754    ///             .retry_policy(
5755    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
5756    ///                     Duration::from_secs(1),
5757    ///                     2,
5758    ///                     Some(Duration::from_secs(30)),
5759    ///                 ),
5760    ///             )
5761    ///             .start_to_close_timeout(Duration::from_secs(60))
5762    ///             .schedule_to_close_timeout(Duration::from_secs(180))
5763    ///             .heartbeat_timeout(Duration::from_secs(15)),
5764    ///         json!([{"order_id": "order-42"}]),
5765    ///     )
5766    ///     .await;
5767    /// match result {
5768    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
5769    ///         "reason": failure.reason,
5770    ///         "timeout_kind": failure.timeout_kind,
5771    ///     })),
5772    ///     other => other,
5773    /// }
5774    /// # }
5775    /// ```
5776    pub fn activity_with_options<T: Serialize>(
5777        &self,
5778        activity_type: impl Into<String>,
5779        options: ActivityOptions,
5780        args: T,
5781    ) -> ActivityCall {
5782        ActivityCall {
5783            ctx: self.clone(),
5784            activity_type: activity_type.into(),
5785            options,
5786            args: Some(AvroValue::from_serialize(&args)),
5787            scheduled: false,
5788        }
5789    }
5790
5791    pub async fn activity_avro_value<T: Serialize>(
5792        &self,
5793        activity_type: impl Into<String>,
5794        args: T,
5795    ) -> Result<AvroValue> {
5796        let mut call = self.activity(activity_type, args);
5797        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5798    }
5799
5800    pub async fn activity_avro_value_with_options<T: Serialize>(
5801        &self,
5802        activity_type: impl Into<String>,
5803        options: ActivityOptions,
5804        args: T,
5805    ) -> Result<AvroValue> {
5806        let mut call = self.activity_with_options(activity_type, options, args);
5807        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5808    }
5809
5810    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5811        SignalCall {
5812            ctx: self.clone(),
5813            signal_name: signal_name.into(),
5814            opened_wait: false,
5815            matched_pending: false,
5816        }
5817    }
5818
5819    pub async fn wait_signal_avro_value(
5820        &self,
5821        signal_name: impl Into<String>,
5822    ) -> Result<Vec<AvroValue>> {
5823        let mut call = self.wait_signal(signal_name);
5824        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5825    }
5826
5827    /// Wait for server-backed durable time without blocking the worker executor.
5828    ///
5829    /// Polling this future emits one `start_timer` command and yields. The
5830    /// server records the deadline, so neither worker nor server restarts reset
5831    /// the wait. Replay resolves the future only from a `TimerScheduled` and
5832    /// `TimerFired` pair at the same position in the shared durable-command
5833    /// stream, with matching sequence, timer identity, and delay. Sub-second
5834    /// durations round up because protocol deadlines use whole seconds.
5835    ///
5836    /// ```no_run
5837    /// # use durable_workflow::{json, Client, Worker};
5838    /// # use std::time::Duration;
5839    /// # fn configure(client: Client) {
5840    /// let mut worker = Worker::new(client, "rust-workers");
5841    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
5842    ///     ctx.sleep(Duration::from_secs(5)).await?;
5843    ///     Ok(json!({"status": "timer fired"}))
5844    /// });
5845    /// # }
5846    /// ```
5847    pub fn sleep(&self, duration: Duration) -> TimerCall {
5848        let delay_seconds = duration
5849            .as_secs()
5850            .checked_add(u64::from(duration.subsec_nanos() > 0));
5851        TimerCall {
5852            ctx: self.clone(),
5853            delay_seconds,
5854            scheduled: false,
5855            matched_pending: false,
5856        }
5857    }
5858
5859    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
5860    pub fn start_timer(&self, duration: Duration) -> TimerCall {
5861        self.sleep(duration)
5862    }
5863
5864    /// Evaluate a non-deterministic callback once and durably record its typed value.
5865    ///
5866    /// On replay the callback is not invoked: the value is decoded from the
5867    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
5868    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
5869    /// small values that must remain fixed for the lifetime of a workflow run.
5870    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5871    where
5872        T: Serialize + DeserializeOwned,
5873        F: FnOnce() -> T,
5874    {
5875        {
5876            let mut state = self
5877                .state
5878                .lock()
5879                .map_err(|_| Error::WorkflowStatePoisoned)?;
5880            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5881                return match recorded {
5882                    RecordedCommand::SideEffect { sequence, value } => {
5883                        state.command_cursor += 1;
5884                        value.deserialize().map_err(|error| {
5885                            Error::NonDeterministicReplay(ReplayFailure::new(
5886                                "side_effect_type_mismatch",
5887                                Some(sequence),
5888                                Some(std::any::type_name::<T>().to_string()),
5889                                Some(error.to_string()),
5890                                "recorded side-effect value is incompatible with the requested Rust type",
5891                            ))
5892                        })
5893                    }
5894                    other => Err(command_mismatch(&other, "side effect")),
5895                };
5896            }
5897        }
5898
5899        let value = callback();
5900        let avro_value = AvroValue::from_serialize(&value)?;
5901        let mut state = self
5902            .state
5903            .lock()
5904            .map_err(|_| Error::WorkflowStatePoisoned)?;
5905        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5906        state.commands.push(json!({
5907            "type": "record_side_effect",
5908            "result": result,
5909        }));
5910        Ok(value)
5911    }
5912
5913    /// Record or replay a lossless fixed Avro Value side effect.
5914    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5915    where
5916        F: FnOnce() -> AvroValue,
5917    {
5918        {
5919            let mut state = self
5920                .state
5921                .lock()
5922                .map_err(|_| Error::WorkflowStatePoisoned)?;
5923            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5924                return match recorded {
5925                    RecordedCommand::SideEffect { value, .. } => {
5926                        state.command_cursor += 1;
5927                        Ok(value)
5928                    }
5929                    other => Err(command_mismatch(&other, "side effect")),
5930                };
5931            }
5932        }
5933
5934        let value = callback();
5935        let mut state = self
5936            .state
5937            .lock()
5938            .map_err(|_| Error::WorkflowStatePoisoned)?;
5939        let result = encode_typed_envelope(&value, &state.payload_codec)?;
5940        state.commands.push(json!({
5941            "type": "record_side_effect",
5942            "result": result,
5943        }));
5944        Ok(value)
5945    }
5946
5947    /// Record a UUIDv4 once and return the same UUID on every replay.
5948    pub fn uuid_v4(&self) -> Result<Uuid> {
5949        self.side_effect(Uuid::new_v4)
5950    }
5951
5952    /// Select the newest supported version for a change, or replay the version
5953    /// already committed for that stable change ID.
5954    pub fn get_version(
5955        &self,
5956        change_id: impl Into<String>,
5957        min_supported: i32,
5958        max_supported: i32,
5959    ) -> Result<i32> {
5960        let change_id = change_id.into();
5961        if change_id.trim().is_empty() {
5962            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5963                "version_change_id_invalid",
5964                None,
5965                Some("non-empty change ID".to_string()),
5966                Some(change_id),
5967                "version markers require a stable non-empty change ID",
5968            )));
5969        }
5970        if min_supported > max_supported {
5971            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5972                "version_range_invalid",
5973                None,
5974                Some("min_supported <= max_supported".to_string()),
5975                Some(format!("{min_supported}..={max_supported}")),
5976                "version marker supported range is invalid",
5977            )));
5978        }
5979
5980        let mut state = self
5981            .state
5982            .lock()
5983            .map_err(|_| Error::WorkflowStatePoisoned)?;
5984        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5985            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5986            return Ok(version);
5987        }
5988
5989        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5990            return match recorded {
5991                RecordedCommand::VersionMarker {
5992                    sequence,
5993                    change_id: recorded_change_id,
5994                    version,
5995                    ..
5996                } => {
5997                    if recorded_change_id != change_id {
5998                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5999                            "version_change_id_mismatch",
6000                            Some(sequence),
6001                            Some(recorded_change_id),
6002                            Some(change_id),
6003                            "recorded version marker change ID differs from current workflow code",
6004                        )));
6005                    }
6006                    ensure_version_supported(
6007                        &change_id,
6008                        version,
6009                        min_supported,
6010                        max_supported,
6011                        sequence,
6012                    )?;
6013                    state.command_cursor += 1;
6014                    state.version_markers.insert(change_id, (version, sequence));
6015                    Ok(version)
6016                }
6017                other => Err(command_mismatch(
6018                    &other,
6019                    format!("version marker:{change_id}"),
6020                )),
6021            };
6022        }
6023
6024        let version = max_supported;
6025        state.commands.push(json!({
6026            "type": "record_version_marker",
6027            "change_id": change_id,
6028            "version": version,
6029            "min_supported": min_supported,
6030            "max_supported": max_supported,
6031        }));
6032        // Sequence numbers are assigned by the server. Zero identifies a marker
6033        // selected in this uncommitted decision batch for duplicate-call checks.
6034        state.version_markers.insert(change_id, (version, 0));
6035        Ok(version)
6036    }
6037
6038    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
6039    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
6040        Ok(self.get_version(change_id, -1, 1)? == 1)
6041    }
6042
6043    /// Keep a patch marker in history after the legacy branch has been removed.
6044    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
6045        self.get_version(change_id, -1, 1).map(|_| ())
6046    }
6047
6048    /// Start a named durable child on an explicit queue and await its result.
6049    ///
6050    /// The command is recorded in the parent's sequence-ordered durable command
6051    /// stream. Replay keeps a scheduled child pending without emitting another
6052    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
6053    /// values preserve the history payload codec and include both sides of the
6054    /// durable relationship; failures are returned as
6055    /// [`Error::ChildWorkflowFailed`].
6056    ///
6057    /// ```no_run
6058    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
6059    /// # fn configure(client: Client) {
6060    /// let mut worker = Worker::new(client, "parent-workers");
6061    /// worker.register_workflow("order-parent", |ctx, _input| async move {
6062    ///     let child = ctx
6063    ///         .start_child_workflow(
6064    ///             "fulfil-order",
6065    ///             ChildWorkflowOptions::new("fulfilment-workers")
6066    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
6067    ///             json!([{"order_id": "order-42"}]),
6068    ///         )
6069    ///         .await?;
6070    ///     Ok(child.result)
6071    /// });
6072    /// # }
6073    /// ```
6074    pub fn start_child_workflow<T: Serialize>(
6075        &self,
6076        workflow_type: impl Into<String>,
6077        options: ChildWorkflowOptions,
6078        args: T,
6079    ) -> ChildWorkflowCall {
6080        ChildWorkflowCall {
6081            ctx: self.clone(),
6082            workflow_type: workflow_type.into(),
6083            options,
6084            args: Some(AvroValue::from_serialize(&args)),
6085            scheduled: false,
6086            matched_pending: false,
6087        }
6088    }
6089
6090    pub async fn start_child_workflow_avro_value<T: Serialize>(
6091        &self,
6092        workflow_type: impl Into<String>,
6093        options: ChildWorkflowOptions,
6094        args: T,
6095    ) -> Result<ChildWorkflowAvroResult> {
6096        let mut call = self.start_child_workflow(workflow_type, options, args);
6097        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6098    }
6099
6100    fn take_commands(&self) -> Result<Vec<Value>> {
6101        let mut state = self
6102            .state
6103            .lock()
6104            .map_err(|_| Error::WorkflowStatePoisoned)?;
6105        Ok(std::mem::take(&mut state.commands))
6106    }
6107
6108    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6109        let mut state = self
6110            .state
6111            .lock()
6112            .map_err(|_| Error::WorkflowStatePoisoned)?;
6113
6114        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6115            return Err(command_mismatch(&recorded, "continue as new"));
6116        }
6117        if state.recorded_continue_as_new_sequence.is_some() {
6118            state.continue_as_new_consumed = true;
6119            return Ok(None);
6120        }
6121
6122        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6123        let mut command = serde_json::Map::from_iter([
6124            ("type".to_string(), json!("continue_as_new")),
6125            ("arguments".to_string(), arguments),
6126            ("queue".to_string(), json!(state.task_queue.clone())),
6127        ]);
6128        if let Some(workflow_type) = request.options.workflow_type {
6129            command.insert("workflow_type".to_string(), json!(workflow_type));
6130        }
6131        if let Some(task_queue) = request.options.task_queue {
6132            command.insert("queue".to_string(), json!(task_queue));
6133        }
6134        Ok(Some(Value::Object(command)))
6135    }
6136
6137    fn matched_recorded_pending(&self) -> Result<bool> {
6138        let state = self
6139            .state
6140            .lock()
6141            .map_err(|_| Error::WorkflowStatePoisoned)?;
6142        Ok(state.matched_recorded_pending)
6143    }
6144
6145    fn ensure_history_consumed(&self) -> Result<()> {
6146        let state = self
6147            .state
6148            .lock()
6149            .map_err(|_| Error::WorkflowStatePoisoned)?;
6150        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6151            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6152                "recorded_commands_unconsumed",
6153                Some(command.sequence()),
6154                Some(command.shape().to_string()),
6155                Some("workflow completion".to_string()),
6156                "workflow completed before consuming all recorded durable commands",
6157            )));
6158        }
6159        if let Some(sequence) = state
6160            .recorded_continue_as_new_sequence
6161            .filter(|_| !state.continue_as_new_consumed)
6162        {
6163            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6164                "recorded_continue_as_new_unconsumed",
6165                Some(sequence),
6166                Some("continue as new".to_string()),
6167                Some("workflow completion".to_string()),
6168                "workflow completed without consuming its recorded continue-as-new transition",
6169            )));
6170        }
6171        Ok(())
6172    }
6173}
6174
6175#[derive(Debug)]
6176struct WorkflowState {
6177    workflow_id: Option<String>,
6178    run_id: Option<String>,
6179    task_queue: String,
6180    payload_codec: String,
6181    history_budget: WorkflowHistoryBudget,
6182    resume_signal: Option<ResumeSignal>,
6183    recorded_commands: Vec<RecordedCommand>,
6184    recorded_continue_as_new_sequence: Option<u64>,
6185    continue_as_new_consumed: bool,
6186    command_cursor: usize,
6187    matched_recorded_pending: bool,
6188    version_markers: HashMap<String, (i32, u64)>,
6189    commands: Vec<Value>,
6190}
6191
6192impl WorkflowState {
6193    #[cfg(test)]
6194    fn new(
6195        history: Vec<HistoryEvent>,
6196        task_queue: String,
6197        payload_codec: String,
6198        resume_signal: Option<ResumeSignal>,
6199    ) -> Result<Self> {
6200        Self::new_with_identity(
6201            history,
6202            None,
6203            None,
6204            task_queue,
6205            payload_codec,
6206            resume_signal,
6207        )
6208    }
6209
6210    fn new_with_identity(
6211        history: Vec<HistoryEvent>,
6212        workflow_id: Option<String>,
6213        run_id: Option<String>,
6214        task_queue: String,
6215        payload_codec: String,
6216        resume_signal: Option<ResumeSignal>,
6217    ) -> Result<Self> {
6218        let recorded_commands = recorded_commands(
6219            &history,
6220            &payload_codec,
6221            WorkflowIdentity {
6222                workflow_id: workflow_id.clone(),
6223                run_id: run_id.clone(),
6224            },
6225        )?;
6226        let recorded_continue_as_new = history
6227            .iter()
6228            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6229            .collect::<Vec<_>>();
6230        if recorded_continue_as_new.len() > 1 {
6231            return Err(invalid_recorded_history(
6232                "duplicate_continue_as_new_transition",
6233                recorded_continue_as_new
6234                    .last()
6235                    .and_then(|event| durable_event_sequence(event))
6236                    .unwrap_or(0),
6237                "one WorkflowContinuedAsNew event",
6238                &format!(
6239                    "{} WorkflowContinuedAsNew events",
6240                    recorded_continue_as_new.len()
6241                ),
6242                "workflow history records one continue-as-new transition more than once",
6243            ));
6244        }
6245        let recorded_continue_as_new_sequence = recorded_continue_as_new
6246            .first()
6247            .map(|event| {
6248                durable_event_sequence(event).ok_or_else(|| {
6249                    Error::NonDeterministicReplay(ReplayFailure::new(
6250                        "continue_as_new_sequence_missing",
6251                        None,
6252                        Some("recorded transition sequence".to_string()),
6253                        Some("missing sequence".to_string()),
6254                        "WorkflowContinuedAsNew history is missing its recorded sequence",
6255                    ))
6256                })
6257            })
6258            .transpose()?;
6259        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6260        Ok(Self {
6261            workflow_id,
6262            run_id,
6263            task_queue,
6264            payload_codec,
6265            history_budget: WorkflowHistoryBudget {
6266                event_count,
6267                ..WorkflowHistoryBudget::default()
6268            },
6269            resume_signal,
6270            recorded_commands,
6271            recorded_continue_as_new_sequence,
6272            continue_as_new_consumed: false,
6273            command_cursor: 0,
6274            matched_recorded_pending: false,
6275            version_markers: HashMap::new(),
6276            commands: Vec::new(),
6277        })
6278    }
6279}
6280
6281#[derive(Clone, Debug)]
6282enum RecordedCommand {
6283    Activity {
6284        sequence: u64,
6285        activity_type: Option<String>,
6286        options: Option<RecordedActivityOptions>,
6287        outcome: Option<ActivityOutcome>,
6288    },
6289    Timer {
6290        sequence: u64,
6291        delay_seconds: u64,
6292        fired: bool,
6293    },
6294    ChildWorkflow {
6295        sequence: u64,
6296        workflow_type: Option<String>,
6297        outcome: Option<ChildWorkflowOutcome>,
6298    },
6299    SignalWait {
6300        sequence: u64,
6301        signal_name: String,
6302        value: Option<Vec<AvroValue>>,
6303    },
6304    SideEffect {
6305        sequence: u64,
6306        value: AvroValue,
6307    },
6308    VersionMarker {
6309        sequence: u64,
6310        change_id: String,
6311        version: i32,
6312    },
6313}
6314
6315#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6316struct RecordedActivityOptions {
6317    task_queue: RecordedSnapshotValue<Option<String>>,
6318    execution_mode: RecordedSnapshotValue<Option<String>>,
6319    retry_policy: ActivityRetrySnapshot,
6320}
6321
6322#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6323enum RecordedSnapshotValue<T> {
6324    /// Older history did not persist this field, so it cannot constrain replay.
6325    Unknown,
6326    Known(T),
6327}
6328
6329impl<T: PartialEq> RecordedSnapshotValue<T> {
6330    fn matches_current(&self, current: &Self) -> bool {
6331        match self {
6332            Self::Unknown => true,
6333            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6334        }
6335    }
6336}
6337
6338#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6339struct ActivityRetrySnapshot {
6340    snapshot_version: RecordedSnapshotValue<Option<u64>>,
6341    max_attempts: RecordedSnapshotValue<Option<u64>>,
6342    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6343    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6344    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6345    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6346    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6347    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6348}
6349
6350impl ActivityRetrySnapshot {
6351    fn matches_current(&self, current: &Self) -> bool {
6352        self.snapshot_version
6353            .matches_current(&current.snapshot_version)
6354            && self.max_attempts.matches_current(&current.max_attempts)
6355            && self
6356                .backoff_seconds
6357                .matches_current(&current.backoff_seconds)
6358            && self
6359                .start_to_close_timeout
6360                .matches_current(&current.start_to_close_timeout)
6361            && self
6362                .schedule_to_start_timeout
6363                .matches_current(&current.schedule_to_start_timeout)
6364            && self
6365                .schedule_to_close_timeout
6366                .matches_current(&current.schedule_to_close_timeout)
6367            && self
6368                .heartbeat_timeout
6369                .matches_current(&current.heartbeat_timeout)
6370            && self
6371                .non_retryable_error_types
6372                .matches_current(&current.non_retryable_error_types)
6373    }
6374}
6375
6376fn recorded_optional_u64(
6377    object: Option<&serde_json::Map<String, Value>>,
6378    field: &str,
6379) -> RecordedSnapshotValue<Option<u64>> {
6380    match object.and_then(|object| object.get(field)) {
6381        None => RecordedSnapshotValue::Unknown,
6382        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6383        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6384    }
6385}
6386
6387fn recorded_optional_string(
6388    object: &serde_json::Map<String, Value>,
6389    field: &str,
6390) -> RecordedSnapshotValue<Option<String>> {
6391    match object.get(field) {
6392        None => RecordedSnapshotValue::Unknown,
6393        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6394        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6395    }
6396}
6397
6398fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6399    let policy = policy.and_then(Value::as_object);
6400    let backoff_seconds = policy
6401        .and_then(|policy| policy.get("backoff_seconds"))
6402        .and_then(Value::as_array)
6403        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6404        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6405    let mut non_retryable_error_types = Vec::new();
6406    for error_type in policy
6407        .and_then(|policy| policy.get("non_retryable_error_types"))
6408        .and_then(Value::as_array)
6409        .into_iter()
6410        .flatten()
6411        .filter_map(Value::as_str)
6412        .map(str::trim)
6413        .filter(|error_type| !error_type.is_empty())
6414    {
6415        if !non_retryable_error_types
6416            .iter()
6417            .any(|recorded| recorded == error_type)
6418        {
6419            non_retryable_error_types.push(error_type.to_string());
6420        }
6421    }
6422
6423    ActivityRetrySnapshot {
6424        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6425        max_attempts: recorded_optional_u64(policy, "max_attempts"),
6426        backoff_seconds,
6427        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6428        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6429        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6430        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6431        non_retryable_error_types: if policy
6432            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6433        {
6434            RecordedSnapshotValue::Known(non_retryable_error_types)
6435        } else {
6436            RecordedSnapshotValue::Unknown
6437        },
6438    }
6439}
6440
6441fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6442    let policy = options.retry_policy.as_ref();
6443    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6444        Some(Value::Null) => None,
6445        Some(value) => value_as_u64(value),
6446        None => Some(1),
6447    };
6448    let backoff_seconds = policy
6449        .and_then(|policy| policy.get("backoff_seconds"))
6450        .and_then(Value::as_array)
6451        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6452        .unwrap_or_default();
6453    let non_retryable_error_types = policy
6454        .and_then(|policy| policy.get("non_retryable_error_types"))
6455        .and_then(Value::as_array)
6456        .into_iter()
6457        .flatten()
6458        .filter_map(Value::as_str)
6459        .map(str::to_string)
6460        .collect();
6461
6462    ActivityRetrySnapshot {
6463        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6464        max_attempts: RecordedSnapshotValue::Known(max_attempts),
6465        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6466        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6467        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6468        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6469        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6470        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6471    }
6472}
6473
6474fn activity_options_description(options: &RecordedActivityOptions) -> String {
6475    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6476}
6477
6478impl RecordedCommand {
6479    fn sequence(&self) -> u64 {
6480        match self {
6481            Self::Activity { sequence, .. }
6482            | Self::Timer { sequence, .. }
6483            | Self::ChildWorkflow { sequence, .. }
6484            | Self::SignalWait { sequence, .. }
6485            | Self::SideEffect { sequence, .. }
6486            | Self::VersionMarker { sequence, .. } => *sequence,
6487        }
6488    }
6489
6490    fn shape(&self) -> &'static str {
6491        match self {
6492            Self::Activity { .. } => "activity",
6493            Self::Timer { .. } => "timer",
6494            Self::ChildWorkflow { .. } => "child workflow",
6495            Self::SignalWait { .. } => "signal wait",
6496            Self::SideEffect { .. } => "side effect",
6497            Self::VersionMarker { .. } => "version marker",
6498        }
6499    }
6500}
6501
6502fn ensure_version_supported(
6503    change_id: &str,
6504    version: i32,
6505    min_supported: i32,
6506    max_supported: i32,
6507    sequence: u64,
6508) -> Result<()> {
6509    if (min_supported..=max_supported).contains(&version) {
6510        return Ok(());
6511    }
6512    Err(Error::NonDeterministicReplay(ReplayFailure::new(
6513        "version_marker_incompatible_range",
6514        (sequence != 0).then_some(sequence),
6515        Some(format!("{min_supported}..={max_supported}")),
6516        Some(format!("{change_id}:{version}")),
6517        "recorded workflow version is outside the range supported by current code",
6518    )))
6519}
6520
6521#[derive(Clone, Debug)]
6522struct ResumeSignal {
6523    signal_name: String,
6524    arguments: Vec<AvroValue>,
6525}
6526
6527pub struct ActivityCall {
6528    ctx: WorkflowContext,
6529    activity_type: String,
6530    options: ActivityOptions,
6531    args: Option<Result<AvroValue>>,
6532    scheduled: bool,
6533}
6534
6535impl ActivityCall {
6536    fn poll_avro_value(
6537        mut self: Pin<&mut Self>,
6538        _cx: &mut TaskContext<'_>,
6539    ) -> Poll<Result<AvroValue>> {
6540        let ctx = self.ctx.clone();
6541        let mut state = match ctx.state.lock() {
6542            Ok(state) => state,
6543            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6544        };
6545
6546        if self.scheduled {
6547            return Poll::Pending;
6548        }
6549
6550        let options = match self.options.validate() {
6551            Ok(options) => options,
6552            Err(error) => {
6553                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6554            }
6555        };
6556        let task_queue = options
6557            .task_queue
6558            .clone()
6559            .unwrap_or_else(|| state.task_queue.clone());
6560        let current_recorded_options = RecordedActivityOptions {
6561            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6562            // Rust schedules ordinary durable activities. The server records a
6563            // non-null mode only for a specialized execution primitive.
6564            execution_mode: RecordedSnapshotValue::Known(None),
6565            retry_policy: current_activity_retry_snapshot(&options),
6566        };
6567
6568        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6569            let sequence = recorded.sequence();
6570            match recorded {
6571                RecordedCommand::Activity {
6572                    activity_type,
6573                    options: recorded_options,
6574                    outcome,
6575                    ..
6576                } => {
6577                    if let Some(recorded_type) = activity_type {
6578                        if recorded_type != self.activity_type {
6579                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6580                                ReplayFailure::new(
6581                                    "recorded_command_detail_mismatch",
6582                                    Some(sequence),
6583                                    Some(format!("activity:{recorded_type}")),
6584                                    Some(format!("activity:{}", self.activity_type)),
6585                                    "recorded activity type differs from the current workflow command",
6586                                ),
6587                            )));
6588                        }
6589                    }
6590                    if let Some(recorded_options) = recorded_options {
6591                        if !recorded_options
6592                            .task_queue
6593                            .matches_current(&current_recorded_options.task_queue)
6594                        {
6595                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6596                                ReplayFailure::new(
6597                                    "activity_task_queue_mismatch",
6598                                    Some(sequence),
6599                                    Some(activity_options_description(&recorded_options)),
6600                                    Some(activity_options_description(&current_recorded_options)),
6601                                    "recorded activity task queue differs from the current workflow command",
6602                                ),
6603                            )));
6604                        }
6605                        if !recorded_options
6606                            .execution_mode
6607                            .matches_current(&current_recorded_options.execution_mode)
6608                        {
6609                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6610                                ReplayFailure::new(
6611                                    "activity_execution_mode_mismatch",
6612                                    Some(sequence),
6613                                    Some(activity_options_description(&recorded_options)),
6614                                    Some(activity_options_description(&current_recorded_options)),
6615                                    "recorded activity execution mode differs from the current workflow command",
6616                                ),
6617                            )));
6618                        }
6619                        if !recorded_options
6620                            .retry_policy
6621                            .matches_current(&current_recorded_options.retry_policy)
6622                        {
6623                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6624                                ReplayFailure::new(
6625                                    "activity_retry_policy_mismatch",
6626                                    Some(sequence),
6627                                    Some(activity_options_description(&recorded_options)),
6628                                    Some(activity_options_description(&current_recorded_options)),
6629                                    "recorded activity retry policy differs from the current workflow command",
6630                                ),
6631                            )));
6632                        }
6633                    }
6634                    state.command_cursor += 1;
6635                    if let Some(outcome) = outcome {
6636                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6637                    }
6638                    state.matched_recorded_pending = true;
6639                    self.scheduled = true;
6640                    return Poll::Pending;
6641                }
6642                other => {
6643                    return Poll::Ready(Err(command_mismatch(
6644                        &other,
6645                        format!("activity:{}", self.activity_type),
6646                    )));
6647                }
6648            }
6649        }
6650
6651        if !self.scheduled {
6652            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6653                Ok(args) => args,
6654                Err(error) => return Poll::Ready(Err(error)),
6655            };
6656            let arguments = normalize_avro_arguments(args);
6657            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6658                Ok(envelope) => envelope,
6659                Err(error) => return Poll::Ready(Err(error)),
6660            };
6661
6662            let mut command = serde_json::Map::from_iter([
6663                ("type".to_string(), json!("schedule_activity")),
6664                (
6665                    "activity_type".to_string(),
6666                    json!(self.activity_type.clone()),
6667                ),
6668                ("queue".to_string(), json!(task_queue)),
6669                ("arguments".to_string(), envelope),
6670            ]);
6671            for (field, value) in [
6672                ("start_to_close_timeout", options.start_to_close_timeout),
6673                (
6674                    "schedule_to_start_timeout",
6675                    options.schedule_to_start_timeout,
6676                ),
6677                (
6678                    "schedule_to_close_timeout",
6679                    options.schedule_to_close_timeout,
6680                ),
6681                ("heartbeat_timeout", options.heartbeat_timeout),
6682            ] {
6683                if let Some(value) = value {
6684                    command.insert(field.to_string(), json!(value));
6685                }
6686            }
6687            if let Some(retry_policy) = options.retry_policy {
6688                command.insert("retry_policy".to_string(), retry_policy);
6689            }
6690            state.commands.push(Value::Object(command));
6691            self.scheduled = true;
6692        }
6693
6694        Poll::Pending
6695    }
6696}
6697
6698impl Future for ActivityCall {
6699    type Output = Result<Value>;
6700
6701    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6702        match self.poll_avro_value(cx) {
6703            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6704            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6705            Poll::Pending => Poll::Pending,
6706        }
6707    }
6708}
6709
6710/// Future returned by [`WorkflowContext::sleep`].
6711pub struct TimerCall {
6712    ctx: WorkflowContext,
6713    delay_seconds: Option<u64>,
6714    scheduled: bool,
6715    matched_pending: bool,
6716}
6717
6718impl Future for TimerCall {
6719    type Output = Result<()>;
6720
6721    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6722        if self.matched_pending {
6723            return Poll::Pending;
6724        }
6725
6726        let ctx = self.ctx.clone();
6727        let Some(requested_delay) = self.delay_seconds else {
6728            return Poll::Ready(Err(Error::TimerDurationOverflow));
6729        };
6730        let mut state = match ctx.state.lock() {
6731            Ok(state) => state,
6732            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6733        };
6734
6735        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6736            match recorded {
6737                RecordedCommand::Timer {
6738                    sequence,
6739                    delay_seconds,
6740                    fired,
6741                    ..
6742                } => {
6743                    if delay_seconds != requested_delay {
6744                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6745                            ReplayFailure::new(
6746                                "timer_delay_mismatch",
6747                                Some(sequence),
6748                                Some(format!("timer:{delay_seconds}s")),
6749                                Some(format!("timer:{requested_delay}s")),
6750                                "recorded timer delay differs from the current workflow command",
6751                            ),
6752                        )));
6753                    }
6754                    state.command_cursor += 1;
6755                    if fired {
6756                        return Poll::Ready(Ok(()));
6757                    }
6758                    state.matched_recorded_pending = true;
6759                    self.scheduled = true;
6760                    self.matched_pending = true;
6761                    return Poll::Pending;
6762                }
6763                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6764            }
6765        }
6766
6767        if !self.scheduled {
6768            state.commands.push(json!({
6769                "type": "start_timer",
6770                "delay_seconds": requested_delay,
6771            }));
6772            self.scheduled = true;
6773        }
6774
6775        Poll::Pending
6776    }
6777}
6778
6779/// Future returned by [`WorkflowContext::start_child_workflow`].
6780pub struct ChildWorkflowCall {
6781    ctx: WorkflowContext,
6782    workflow_type: String,
6783    options: ChildWorkflowOptions,
6784    args: Option<Result<AvroValue>>,
6785    scheduled: bool,
6786    matched_pending: bool,
6787}
6788
6789impl ChildWorkflowCall {
6790    fn poll_avro_value(
6791        mut self: Pin<&mut Self>,
6792        _cx: &mut TaskContext<'_>,
6793    ) -> Poll<Result<ChildWorkflowAvroResult>> {
6794        if self.matched_pending {
6795            return Poll::Pending;
6796        }
6797
6798        let ctx = self.ctx.clone();
6799        let mut state = match ctx.state.lock() {
6800            Ok(state) => state,
6801            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6802        };
6803
6804        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6805            let sequence = recorded.sequence();
6806            match recorded {
6807                RecordedCommand::ChildWorkflow {
6808                    workflow_type,
6809                    outcome,
6810                    ..
6811                } => {
6812                    if let Some(recorded_type) = workflow_type {
6813                        if recorded_type != self.workflow_type {
6814                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6815                                ReplayFailure::new(
6816                                    "recorded_command_detail_mismatch",
6817                                    Some(sequence),
6818                                    Some(format!("child workflow:{recorded_type}")),
6819                                    Some(format!("child workflow:{}", self.workflow_type)),
6820                                    "recorded child workflow type differs from the current workflow command",
6821                                ),
6822                            )));
6823                        }
6824                    }
6825                    state.command_cursor += 1;
6826                    if let Some(outcome) = outcome {
6827                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6828                    }
6829                    state.matched_recorded_pending = true;
6830                    self.scheduled = true;
6831                    self.matched_pending = true;
6832                    return Poll::Pending;
6833                }
6834                other => {
6835                    return Poll::Ready(Err(command_mismatch(
6836                        &other,
6837                        format!("child workflow:{}", self.workflow_type),
6838                    )));
6839                }
6840            }
6841        }
6842
6843        if !self.scheduled {
6844            if self.options.task_queue.trim().is_empty() {
6845                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6846                    "task_queue must not be empty".to_string(),
6847                )));
6848            }
6849            for (name, value) in [
6850                (
6851                    "execution_timeout_seconds",
6852                    self.options.execution_timeout_seconds,
6853                ),
6854                ("run_timeout_seconds", self.options.run_timeout_seconds),
6855            ] {
6856                if value == Some(0) {
6857                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6858                        "{name} must be at least 1"
6859                    ))));
6860                }
6861            }
6862
6863            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6864                Ok(args) => args,
6865                Err(error) => return Poll::Ready(Err(error)),
6866            };
6867            let arguments = match encode_typed_envelope(
6868                &normalize_avro_arguments(args),
6869                &state.payload_codec,
6870            ) {
6871                Ok(arguments) => arguments,
6872                Err(error) => return Poll::Ready(Err(error)),
6873            };
6874            let mut command = json!({
6875                "type": "start_child_workflow",
6876                "workflow_type": self.workflow_type,
6877                "queue": self.options.task_queue,
6878                "parent_close_policy": self.options.parent_close_policy.as_str(),
6879                "arguments": arguments,
6880            });
6881            let object = command
6882                .as_object_mut()
6883                .expect("child workflow command is always an object");
6884            if let Some(policy) = &self.options.retry_policy {
6885                let mut retry_policy = serde_json::Map::new();
6886                if let Some(max_attempts) = policy.max_attempts {
6887                    if max_attempts == 0 {
6888                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6889                            "retry_policy.max_attempts must be at least 1".to_string(),
6890                        )));
6891                    }
6892                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6893                }
6894                if !policy.backoff_seconds.is_empty() {
6895                    retry_policy
6896                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6897                }
6898                if !policy.non_retryable_error_types.is_empty() {
6899                    retry_policy.insert(
6900                        "non_retryable_error_types".to_string(),
6901                        json!(policy.non_retryable_error_types),
6902                    );
6903                }
6904                if retry_policy.is_empty() {
6905                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6906                        "retry_policy must configure at least one field".to_string(),
6907                    )));
6908                }
6909                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6910            }
6911            if let Some(seconds) = self.options.execution_timeout_seconds {
6912                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6913            }
6914            if let Some(seconds) = self.options.run_timeout_seconds {
6915                object.insert("run_timeout_seconds".to_string(), json!(seconds));
6916            }
6917            state.commands.push(command);
6918            self.scheduled = true;
6919        }
6920
6921        Poll::Pending
6922    }
6923}
6924
6925impl Future for ChildWorkflowCall {
6926    type Output = Result<ChildWorkflowResult>;
6927
6928    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6929        match self.poll_avro_value(cx) {
6930            Poll::Ready(Ok(result)) => match result.result.into_json() {
6931                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6932                    parent: result.parent,
6933                    child: result.child,
6934                    child_workflow_type: result.child_workflow_type,
6935                    result: projected,
6936                })),
6937                Err(error) => Poll::Ready(Err(error)),
6938            },
6939            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6940            Poll::Pending => Poll::Pending,
6941        }
6942    }
6943}
6944
6945fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6946    Error::NonDeterministicReplay(ReplayFailure::new(
6947        "recorded_command_mismatch",
6948        Some(recorded.sequence()),
6949        Some(recorded.shape().to_string()),
6950        Some(actual.into()),
6951        "current workflow command does not match the recorded durable command sequence",
6952    ))
6953}
6954
6955pub struct SignalCall {
6956    ctx: WorkflowContext,
6957    signal_name: String,
6958    opened_wait: bool,
6959    matched_pending: bool,
6960}
6961
6962impl SignalCall {
6963    fn poll_avro_value(
6964        mut self: Pin<&mut Self>,
6965        _cx: &mut TaskContext<'_>,
6966    ) -> Poll<Result<Vec<AvroValue>>> {
6967        if self.matched_pending {
6968            return Poll::Pending;
6969        }
6970
6971        let ctx = self.ctx.clone();
6972        let mut state = match ctx.state.lock() {
6973            Ok(state) => state,
6974            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6975        };
6976
6977        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6978            match recorded {
6979                RecordedCommand::SignalWait {
6980                    sequence,
6981                    signal_name,
6982                    value,
6983                } => {
6984                    if signal_name != self.signal_name {
6985                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6986                            ReplayFailure::new(
6987                                "recorded_command_detail_mismatch",
6988                                Some(sequence),
6989                                Some(format!("signal wait:{signal_name}")),
6990                                Some(format!("signal wait:{}", self.signal_name)),
6991                                "recorded signal name differs from the current workflow command",
6992                            ),
6993                        )));
6994                    }
6995
6996                    state.command_cursor += 1;
6997                    if let Some(value) = value {
6998                        return Poll::Ready(Ok(value));
6999                    }
7000                    if state
7001                        .resume_signal
7002                        .as_ref()
7003                        .is_some_and(|signal| signal.signal_name == self.signal_name)
7004                    {
7005                        let signal = state
7006                            .resume_signal
7007                            .take()
7008                            .expect("matching resume signal is present");
7009                        return Poll::Ready(Ok(signal.arguments));
7010                    }
7011
7012                    state.matched_recorded_pending = true;
7013                    self.opened_wait = true;
7014                    self.matched_pending = true;
7015                    return Poll::Pending;
7016                }
7017                other => {
7018                    return Poll::Ready(Err(command_mismatch(
7019                        &other,
7020                        format!("signal wait:{}", self.signal_name),
7021                    )));
7022                }
7023            }
7024        }
7025
7026        if state
7027            .resume_signal
7028            .as_ref()
7029            .is_some_and(|signal| signal.signal_name == self.signal_name)
7030        {
7031            let signal = state
7032                .resume_signal
7033                .take()
7034                .expect("matching resume signal is present");
7035            return Poll::Ready(Ok(signal.arguments));
7036        }
7037
7038        if !self.opened_wait {
7039            state.commands.push(json!({
7040                "type": "open_signal_wait",
7041                "signal_name": self.signal_name
7042            }));
7043            self.opened_wait = true;
7044        }
7045
7046        Poll::Pending
7047    }
7048}
7049
7050impl Future for SignalCall {
7051    type Output = Result<Vec<Value>>;
7052
7053    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
7054        match self.poll_avro_value(cx) {
7055            Poll::Ready(Ok(values)) => Poll::Ready(
7056                values
7057                    .into_iter()
7058                    .map(AvroValue::into_json)
7059                    .collect::<Result<Vec<_>>>(),
7060            ),
7061            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
7062            Poll::Pending => Poll::Pending,
7063        }
7064    }
7065}
7066
7067#[derive(Clone, Debug)]
7068pub struct ActivityContext {
7069    client: Client,
7070    pub task_id: String,
7071    pub activity_attempt_id: String,
7072    pub lease_owner: String,
7073    pub activity_type: String,
7074    pub attempt_number: u64,
7075    pub task_queue: String,
7076    pub worker_id: String,
7077}
7078
7079impl ActivityContext {
7080    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
7081        self.client
7082            .heartbeat_activity_task(
7083                &self.task_id,
7084                &self.activity_attempt_id,
7085                &self.lease_owner,
7086                details,
7087            )
7088            .await
7089    }
7090}
7091
7092fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
7093    validate_payload_codec(codec)?;
7094    match value {
7095        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
7096            value, codec,
7097        )?)),
7098        None => Ok(AvroValue::Array(Vec::new())),
7099    }
7100}
7101
7102fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7103    let Some(signal_name) = task
7104        .signal_name
7105        .as_deref()
7106        .filter(|value| !value.is_empty())
7107    else {
7108        return Ok(None);
7109    };
7110    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
7111    let AvroValue::Array(arguments) = decoded else {
7112        unreachable!("normalize_avro_arguments always returns an array");
7113    };
7114
7115    Ok(Some(ResumeSignal {
7116        signal_name: signal_name.to_string(),
7117        arguments,
7118    }))
7119}
7120
7121fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
7122    validate_payload_codec(&task.payload_codec)?;
7123    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
7124    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
7125    for event in &task.history_events {
7126        validate_history_event_payloads(event, &task.payload_codec)?;
7127    }
7128    Ok(())
7129}
7130
7131fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
7132    validate_payload_codec(&task.payload_codec)?;
7133    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
7134}
7135
7136fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
7137    validate_payload_codec(&task.payload_codec)?;
7138    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
7139    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
7140    for event in &task.history_events {
7141        validate_history_event_payloads(event, &task.payload_codec)?;
7142    }
7143
7144    let Some(export) = task.history_export.as_ref() else {
7145        return Ok(());
7146    };
7147    let export_codec = match export.get("payloads") {
7148        Some(payloads) => declared_payload_codec(payloads, "codec")?,
7149        None => None,
7150    }
7151    .unwrap_or(&task.payload_codec);
7152    validate_payload_codec(export_codec)?;
7153
7154    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
7155        for event in events {
7156            let event_type = event
7157                .get("event_type")
7158                .or_else(|| event.get("type"))
7159                .and_then(Value::as_str)
7160                .unwrap_or_default();
7161            if let Some(payload) = event.get("payload") {
7162                validate_history_payloads(event_type, payload, export_codec)?;
7163            }
7164        }
7165    }
7166    for signal in export
7167        .get("signals")
7168        .and_then(Value::as_array)
7169        .into_iter()
7170        .flatten()
7171    {
7172        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
7173        validate_payload_codec(codec)?;
7174        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
7175    }
7176    for activity in export
7177        .get("activities")
7178        .and_then(Value::as_array)
7179        .into_iter()
7180        .flatten()
7181    {
7182        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
7183        validate_payload_codec(codec)?;
7184        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
7185        validate_optional_inbound_payload(activity.get("result"), codec)?;
7186    }
7187    Ok(())
7188}
7189
7190fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
7191    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
7192}
7193
7194fn validate_history_payloads(
7195    event_type: &str,
7196    payload: &Value,
7197    fallback_codec: &str,
7198) -> Result<()> {
7199    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
7200    validate_payload_codec(codec)?;
7201    for field in history_payload_fields(event_type) {
7202        validate_optional_inbound_payload(payload.get(*field), codec)?;
7203    }
7204    Ok(())
7205}
7206
7207const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
7208
7209fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
7210    match event_type {
7211        "ActivityCompleted" => &["result"],
7212        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
7213        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
7214        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
7215        "ChildRunCompleted" => &["result", "output"],
7216        "WorkflowCompleted" => &["output"],
7217        "ServiceCallStarted"
7218        | "ServiceCallCompleted"
7219        | "ServiceCallFailed"
7220        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
7221        _ => &[],
7222    }
7223}
7224
7225fn signal_history_payload(payload: &Value) -> Option<&Value> {
7226    SIGNAL_HISTORY_PAYLOAD_FIELDS
7227        .iter()
7228        .find_map(|field| payload.get(*field))
7229}
7230
7231fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
7232    match value.get(field) {
7233        None => Ok(None),
7234        Some(Value::String(codec)) => Ok(Some(codec)),
7235        Some(_) => Err(invalid_payload_envelope()),
7236    }
7237}
7238
7239fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
7240    validate_payload_codec(codec)?;
7241    if let Some(value) = value.filter(|value| !value.is_null()) {
7242        decode_wire_avro_value(value, codec)?;
7243    }
7244    Ok(())
7245}
7246
7247fn recorded_commands(
7248    events: &[HistoryEvent],
7249    fallback_codec: &str,
7250    parent: WorkflowIdentity,
7251) -> Result<Vec<RecordedCommand>> {
7252    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7253    let mut last_new_sequence = None;
7254
7255    for event in events {
7256        let is_activity = matches!(
7257            event.event_type.as_str(),
7258            "ActivityScheduled"
7259                | "ActivityStarted"
7260                | "ActivityHeartbeatRecorded"
7261                | "ActivityRetryScheduled"
7262                | "ActivityCompleted"
7263                | "ActivityFailed"
7264                | "ActivityCancelled"
7265                | "ActivityTimedOut"
7266        );
7267        let is_workflow_timer = matches!(
7268            event.event_type.as_str(),
7269            "TimerScheduled" | "TimerCancelled" | "TimerFired"
7270        ) && !is_internal_timer_event(event);
7271        let is_child_workflow = matches!(
7272            event.event_type.as_str(),
7273            "ChildWorkflowScheduled"
7274                | "ChildRunCompleted"
7275                | "ChildRunFailed"
7276                | "ChildRunCancelled"
7277                | "ChildRunTerminated"
7278        );
7279        let is_signal_wait = is_recorded_signal_wait_event(event);
7280        let is_side_effect = event.event_type == "SideEffectRecorded";
7281        let is_version_marker = event.event_type == "VersionMarkerRecorded";
7282        if !is_activity
7283            && !is_workflow_timer
7284            && !is_child_workflow
7285            && !is_signal_wait
7286            && !is_side_effect
7287            && !is_version_marker
7288        {
7289            continue;
7290        }
7291
7292        let sequence = durable_event_sequence(event).ok_or_else(|| {
7293            Error::NonDeterministicReplay(ReplayFailure::new(
7294                "durable_command_sequence_missing",
7295                None,
7296                Some("positive workflow sequence".to_string()),
7297                Some(event.event_type.clone()),
7298                "durable command history event has no workflow sequence",
7299            ))
7300        })?;
7301        if sequence == 0 {
7302            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7303                "durable_command_sequence_invalid",
7304                Some(sequence),
7305                Some("positive workflow sequence".to_string()),
7306                Some(sequence.to_string()),
7307                "durable command history uses an invalid workflow sequence",
7308            )));
7309        }
7310        if !events_by_sequence.contains_key(&sequence) {
7311            if let Some(previous) = last_new_sequence {
7312                if sequence < previous {
7313                    return Err(invalid_recorded_history(
7314                        "durable_command_sequence_mismatch",
7315                        sequence,
7316                        &format!("workflow sequence greater than {previous}"),
7317                        &sequence.to_string(),
7318                        "durable commands are not strictly ordered by their recorded workflow sequence",
7319                    ));
7320                }
7321            }
7322            last_new_sequence = Some(sequence);
7323        }
7324        events_by_sequence.entry(sequence).or_default().push(event);
7325    }
7326
7327    let commands: Vec<RecordedCommand> = events_by_sequence
7328        .into_iter()
7329        .map(|(sequence, sequence_events)| {
7330            let activity_events: Vec<_> = sequence_events
7331                .iter()
7332                .copied()
7333                .filter(|event| event.event_type.starts_with("Activity"))
7334                .collect();
7335            let timer_events: Vec<_> = sequence_events
7336                .iter()
7337                .copied()
7338                .filter(|event| event.event_type.starts_with("Timer"))
7339                .collect();
7340            let child_events: Vec<_> = sequence_events
7341                .iter()
7342                .copied()
7343                .filter(|event| {
7344                    event.event_type == "ChildWorkflowScheduled"
7345                        || event.event_type.starts_with("ChildRun")
7346                })
7347                .collect();
7348            let signal_wait_events: Vec<_> = sequence_events
7349                .iter()
7350                .copied()
7351                .filter(|event| is_recorded_signal_wait_event(event))
7352                .collect();
7353            let side_effect_events: Vec<_> = sequence_events
7354                .iter()
7355                .copied()
7356                .filter(|event| event.event_type == "SideEffectRecorded")
7357                .collect();
7358            let version_marker_events: Vec<_> = sequence_events
7359                .iter()
7360                .copied()
7361                .filter(|event| event.event_type == "VersionMarkerRecorded")
7362                .collect();
7363
7364            let command_kind_count = usize::from(!activity_events.is_empty())
7365                + usize::from(!timer_events.is_empty())
7366                + usize::from(!child_events.is_empty())
7367                + usize::from(!signal_wait_events.is_empty())
7368                + usize::from(!side_effect_events.is_empty())
7369                + usize::from(!version_marker_events.is_empty());
7370            if command_kind_count > 1 {
7371                let actual = [
7372                    (!activity_events.is_empty()).then_some("activity"),
7373                    (!timer_events.is_empty()).then_some("timer"),
7374                    (!child_events.is_empty()).then_some("child workflow"),
7375                    (!signal_wait_events.is_empty()).then_some("signal wait"),
7376                    (!side_effect_events.is_empty()).then_some("side effect"),
7377                    (!version_marker_events.is_empty()).then_some("version marker"),
7378                ]
7379                .into_iter()
7380                .flatten()
7381                .collect::<Vec<_>>()
7382                .join(" and ");
7383                return Err(invalid_recorded_history(
7384                    "durable_command_sequence_collision",
7385                    sequence,
7386                    "one durable command kind",
7387                    &actual,
7388                    "one workflow sequence records more than one durable command kind",
7389                ));
7390            }
7391
7392            if !activity_events.is_empty() {
7393                let scheduled_count = activity_events
7394                    .iter()
7395                    .filter(|event| event.event_type == "ActivityScheduled")
7396                    .count();
7397                if scheduled_count > 1 {
7398                    return Err(invalid_recorded_history(
7399                        "duplicate_activity_schedule",
7400                        sequence,
7401                        "at most one ActivityScheduled event",
7402                        "multiple ActivityScheduled events",
7403                        "activity history schedules more than one command at one workflow sequence",
7404                    ));
7405                }
7406                let activity_type = activity_events.iter().find_map(|event| {
7407                    event
7408                        .payload
7409                        .get("activity_type")
7410                        .or_else(|| event.payload.get("activity_name"))
7411                        .and_then(Value::as_str)
7412                        .map(str::to_string)
7413                });
7414                if activity_events.iter().filter_map(|event| {
7415                    event
7416                        .payload
7417                        .get("activity_type")
7418                        .or_else(|| event.payload.get("activity_name"))
7419                        .and_then(Value::as_str)
7420                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7421                    return Err(invalid_recorded_history(
7422                        "activity_identity_mismatch",
7423                        sequence,
7424                        activity_type.as_deref().unwrap_or("one activity identity"),
7425                        "conflicting activity identities",
7426                        "activity lifecycle events at one workflow sequence disagree on identity",
7427                    ));
7428                }
7429                let terminal: Vec<_> = activity_events
7430                    .iter()
7431                    .copied()
7432                    .filter(|event| {
7433                        matches!(
7434                            event.event_type.as_str(),
7435                            "ActivityCompleted"
7436                                | "ActivityFailed"
7437                                | "ActivityCancelled"
7438                                | "ActivityTimedOut"
7439                        )
7440                    })
7441                    .collect();
7442                if terminal.len() > 1 {
7443                    return Err(invalid_recorded_history(
7444                        "duplicate_activity_terminal_event",
7445                        sequence,
7446                        "at most one terminal activity event",
7447                        "multiple terminal activity events",
7448                        "activity history settles one command more than once",
7449                    ));
7450                }
7451                let outcome = terminal
7452                    .first()
7453                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7454                    .transpose()?;
7455                let options = activity_events
7456                    .iter()
7457                    .find(|event| event.event_type == "ActivityScheduled")
7458                    .and_then(|event| event.payload.get("activity"))
7459                    .and_then(Value::as_object)
7460                    .map(|activity| RecordedActivityOptions {
7461                        task_queue: recorded_optional_string(activity, "queue"),
7462                        execution_mode: recorded_optional_string(activity, "execution_mode"),
7463                        retry_policy: recorded_activity_retry_snapshot(
7464                            activity.get("retry_policy"),
7465                        ),
7466                    });
7467                return Ok(RecordedCommand::Activity {
7468                    sequence,
7469                    activity_type,
7470                    options,
7471                    outcome,
7472                });
7473            }
7474
7475            if !child_events.is_empty() {
7476                let scheduled: Vec<_> = child_events
7477                    .iter()
7478                    .copied()
7479                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
7480                    .collect();
7481                if scheduled.len() != 1 {
7482                    return Err(invalid_recorded_history(
7483                        "child_workflow_schedule_missing_or_duplicate",
7484                        sequence,
7485                        "one ChildWorkflowScheduled event",
7486                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7487                        "child workflow replay requires exactly one recorded schedule event",
7488                    ));
7489                }
7490                let workflow_type = child_events.iter().find_map(|event| {
7491                    event
7492                        .payload
7493                        .get("child_workflow_type")
7494                        .or_else(|| event.payload.get("workflow_type"))
7495                        .and_then(Value::as_str)
7496                        .filter(|value| !value.is_empty())
7497                        .map(str::to_string)
7498                });
7499                if child_events
7500                    .iter()
7501                    .filter_map(|event| {
7502                        event
7503                            .payload
7504                            .get("child_workflow_type")
7505                            .or_else(|| event.payload.get("workflow_type"))
7506                            .and_then(Value::as_str)
7507                    })
7508                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
7509                {
7510                    return Err(invalid_recorded_history(
7511                        "child_workflow_identity_mismatch",
7512                        sequence,
7513                        workflow_type
7514                            .as_deref()
7515                            .unwrap_or("one child workflow type"),
7516                        "conflicting child workflow types",
7517                        "child workflow lifecycle events at one sequence disagree on type",
7518                    ));
7519                }
7520                let mut outcomes = child_workflow_outcomes(
7521                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7522                    fallback_codec,
7523                    parent.clone(),
7524                )?;
7525                if outcomes.len() > 1 {
7526                    return Err(invalid_recorded_history(
7527                        "duplicate_child_workflow_terminal_event",
7528                        sequence,
7529                        "at most one terminal child event",
7530                        "multiple terminal child events",
7531                        "child workflow history settles one command more than once",
7532                    ));
7533                }
7534                return Ok(RecordedCommand::ChildWorkflow {
7535                    sequence,
7536                    workflow_type,
7537                    outcome: outcomes.pop(),
7538                });
7539            }
7540
7541            if !signal_wait_events.is_empty() {
7542                let opened: Vec<_> = signal_wait_events
7543                    .iter()
7544                    .copied()
7545                    .filter(|event| event.event_type == "SignalWaitOpened")
7546                    .collect();
7547                if opened.len() != 1 {
7548                    return Err(invalid_recorded_history(
7549                        "signal_wait_open_missing_or_duplicate",
7550                        sequence,
7551                        "one SignalWaitOpened event",
7552                        &format!("{} SignalWaitOpened events", opened.len()),
7553                        "signal replay requires exactly one canonical wait-open event",
7554                    ));
7555                }
7556
7557                let applied: Vec<_> = signal_wait_events
7558                    .iter()
7559                    .copied()
7560                    .filter(|event| event.event_type == "SignalApplied")
7561                    .collect();
7562                if applied.len() > 1 {
7563                    return Err(invalid_recorded_history(
7564                        "duplicate_signal_wait_apply",
7565                        sequence,
7566                        "at most one SignalApplied event",
7567                        "multiple SignalApplied events",
7568                        "signal history applies one durable wait more than once",
7569                    ));
7570                }
7571
7572                let signal_names = signal_wait_events
7573                    .iter()
7574                    .map(|event| required_signal_wait_name(event, sequence))
7575                    .collect::<Result<Vec<_>>>()?;
7576                let signal_name = signal_names
7577                    .first()
7578                    .expect("signal wait events are not empty")
7579                    .clone();
7580                if signal_names.iter().any(|candidate| candidate != &signal_name) {
7581                    return Err(invalid_recorded_history(
7582                        "signal_wait_identity_mismatch",
7583                        sequence,
7584                        &signal_name,
7585                        "conflicting signal names",
7586                        "signal wait lifecycle events at one workflow sequence disagree on identity",
7587                    ));
7588                }
7589                let value = applied
7590                    .first()
7591                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
7592                    .transpose()?;
7593                return Ok(RecordedCommand::SignalWait {
7594                    sequence,
7595                    signal_name,
7596                    value,
7597                });
7598            }
7599
7600            if !side_effect_events.is_empty() {
7601                if side_effect_events.len() != 1 {
7602                    return Err(invalid_recorded_history(
7603                        "duplicate_side_effect_record",
7604                        sequence,
7605                        "one SideEffectRecorded event",
7606                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
7607                        "side-effect history records one workflow command more than once",
7608                    ));
7609                }
7610                let event = side_effect_events[0];
7611                let result = event.payload.get("result").ok_or_else(|| {
7612                    invalid_recorded_history(
7613                        "side_effect_result_missing",
7614                        sequence,
7615                        "recorded result payload",
7616                        "missing result",
7617                        "side-effect history is missing its recorded value",
7618                    )
7619                })?;
7620                let has_published_envelope = result.as_str().is_some()
7621                    || result.as_object().is_some_and(|envelope| {
7622                        envelope.get("codec").and_then(Value::as_str).is_some()
7623                            && envelope.get("blob").and_then(Value::as_str).is_some()
7624                    });
7625                if !has_published_envelope {
7626                    return Err(invalid_recorded_history(
7627                        "side_effect_payload_malformed",
7628                        sequence,
7629                        "payload blob or {codec, blob} envelope",
7630                        &result.to_string(),
7631                        "side-effect history result does not use a published payload envelope",
7632                    ));
7633                }
7634                let codec = event
7635                    .payload
7636                    .get("payload_codec")
7637                    .and_then(Value::as_str)
7638                    .unwrap_or(fallback_codec);
7639                let value = decode_wire_avro_value(result, codec).map_err(|error| {
7640                    if error.to_string().contains("unsupported_payload_codec") {
7641                        return error;
7642                    }
7643
7644                    invalid_recorded_history(
7645                        "side_effect_payload_incompatible",
7646                        sequence,
7647                        &format!("valid {codec} payload envelope"),
7648                        &error.to_string(),
7649                        "side-effect history payload cannot be decoded with its recorded codec",
7650                    )
7651                })?;
7652                return Ok(RecordedCommand::SideEffect { sequence, value });
7653            }
7654
7655            if !version_marker_events.is_empty() {
7656                if version_marker_events.len() != 1 {
7657                    return Err(invalid_recorded_history(
7658                        "duplicate_version_marker_record",
7659                        sequence,
7660                        "one VersionMarkerRecorded event",
7661                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7662                        "version-marker history records one workflow command more than once",
7663                    ));
7664                }
7665                let payload = &version_marker_events[0].payload;
7666                let change_id = payload
7667                    .get("change_id")
7668                    .and_then(Value::as_str)
7669                    .filter(|value| !value.is_empty())
7670                    .map(str::to_string)
7671                    .ok_or_else(|| {
7672                        invalid_recorded_history(
7673                            "version_marker_field_missing",
7674                            sequence,
7675                            "non-empty change_id",
7676                            "missing or invalid change_id",
7677                            "version-marker history is missing its stable change ID",
7678                        )
7679                    })?;
7680                let version = required_version_i32(payload, "version", sequence)?;
7681                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7682                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7683                if min_supported > max_supported || version < min_supported || version > max_supported {
7684                    return Err(invalid_recorded_history(
7685                        "version_marker_history_range_invalid",
7686                        sequence,
7687                        "min_supported <= version <= max_supported",
7688                        &format!("{min_supported} <= {version} <= {max_supported}"),
7689                        "recorded version marker contains an internally incompatible range",
7690                    ));
7691                }
7692                return Ok(RecordedCommand::VersionMarker {
7693                    sequence,
7694                    change_id,
7695                    version,
7696                });
7697            }
7698
7699            let scheduled: Vec<_> = timer_events
7700                .iter()
7701                .copied()
7702                .filter(|event| event.event_type == "TimerScheduled")
7703                .collect();
7704            let fired: Vec<_> = timer_events
7705                .iter()
7706                .copied()
7707                .filter(|event| event.event_type == "TimerFired")
7708                .collect();
7709            if scheduled.len() != 1 {
7710                return Err(invalid_recorded_history(
7711                    "timer_schedule_missing_or_duplicate",
7712                    sequence,
7713                    "one TimerScheduled event",
7714                    &format!("{} TimerScheduled events", scheduled.len()),
7715                    "timer replay requires exactly one recorded schedule event",
7716                ));
7717            }
7718            if fired.len() > 1 {
7719                return Err(invalid_recorded_history(
7720                    "duplicate_timer_fire",
7721                    sequence,
7722                    "at most one TimerFired event",
7723                    "multiple TimerFired events",
7724                    "timer history contains more than one fire event for a workflow sequence",
7725                ));
7726            }
7727
7728            let scheduled = scheduled[0];
7729            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7730            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7731            if let Some(fired) = fired.first() {
7732                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7733                if fired_timer_id != timer_id {
7734                    return Err(invalid_recorded_history(
7735                        "timer_identity_mismatch",
7736                        sequence,
7737                        &timer_id,
7738                        &fired_timer_id,
7739                        "TimerFired does not correspond to the recorded TimerScheduled event",
7740                    ));
7741                }
7742                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7743                if fired_delay != delay_seconds {
7744                    return Err(invalid_recorded_history(
7745                        "timer_history_delay_mismatch",
7746                        sequence,
7747                        &delay_seconds.to_string(),
7748                        &fired_delay.to_string(),
7749                        "TimerScheduled and TimerFired record different delays",
7750                    ));
7751                }
7752            }
7753
7754            Ok(RecordedCommand::Timer {
7755                sequence,
7756                delay_seconds,
7757                fired: !fired.is_empty(),
7758            })
7759        })
7760        .collect::<Result<_>>()?;
7761
7762    let mut marker_sequences = HashMap::new();
7763    for command in &commands {
7764        if let RecordedCommand::VersionMarker {
7765            sequence,
7766            change_id,
7767            ..
7768        } = command
7769        {
7770            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7771                return Err(invalid_recorded_history(
7772                    "duplicate_version_marker",
7773                    *sequence,
7774                    &format!("one marker for change ID {change_id:?}"),
7775                    &format!("markers at sequences {first_sequence} and {sequence}"),
7776                    "workflow history contains duplicate markers for one stable change ID",
7777                ));
7778            }
7779        }
7780    }
7781
7782    Ok(commands)
7783}
7784
7785fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7786    payload
7787        .get(field)
7788        .and_then(Value::as_i64)
7789        .and_then(|value| i32::try_from(value).ok())
7790        .ok_or_else(|| {
7791            invalid_recorded_history(
7792                "version_marker_field_missing",
7793                sequence,
7794                &format!("integer {field}"),
7795                "missing or out-of-range integer",
7796                "version-marker history is missing a required integer field",
7797            )
7798        })
7799}
7800
7801fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7802    event
7803        .payload
7804        .get("sequence")
7805        .or_else(|| event.payload.get("workflow_sequence"))
7806        .or_else(|| event.raw.get("sequence"))
7807        .or_else(|| event.raw.get("workflow_sequence"))
7808        .and_then(value_as_u64)
7809}
7810
7811fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7812    matches!(
7813        event
7814            .payload
7815            .get("timer_kind")
7816            .or_else(|| event.raw.get("timer_kind"))
7817            .and_then(Value::as_str),
7818        Some("condition_timeout" | "signal_timeout")
7819    )
7820}
7821
7822fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7823    event
7824        .payload
7825        .get("signal_name")
7826        .or_else(|| event.raw.get("signal_name"))
7827        .and_then(Value::as_str)
7828        .filter(|value| !value.is_empty())
7829        .map(str::to_string)
7830        .ok_or_else(|| {
7831            invalid_recorded_history(
7832                "signal_wait_name_missing",
7833                sequence,
7834                "non-empty signal_name",
7835                &event.event_type,
7836                "canonical signal-wait history is missing its signal identity",
7837            )
7838        })
7839}
7840
7841fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7842    matches!(
7843        event.event_type.as_str(),
7844        "SignalWaitOpened" | "SignalApplied"
7845    )
7846}
7847
7848fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7849    event
7850        .payload
7851        .get(field)
7852        .and_then(Value::as_str)
7853        .filter(|value| !value.is_empty())
7854        .map(str::to_string)
7855        .ok_or_else(|| {
7856            invalid_recorded_history(
7857                "timer_history_field_missing",
7858                sequence,
7859                field,
7860                &event.event_type,
7861                "timer history is missing a required identity field",
7862            )
7863        })
7864}
7865
7866fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7867    event
7868        .payload
7869        .get(field)
7870        .and_then(value_as_u64)
7871        .ok_or_else(|| {
7872            invalid_recorded_history(
7873                "timer_history_field_missing",
7874                sequence,
7875                field,
7876                &event.event_type,
7877                "timer history is missing a required numeric field",
7878            )
7879        })
7880}
7881
7882fn invalid_recorded_history(
7883    reason: &str,
7884    sequence: u64,
7885    expected: &str,
7886    actual: &str,
7887    message: &str,
7888) -> Error {
7889    Error::NonDeterministicReplay(ReplayFailure::new(
7890        reason,
7891        Some(sequence),
7892        Some(expected.to_string()),
7893        Some(actual.to_string()),
7894        message,
7895    ))
7896}
7897
7898type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7899
7900fn activity_outcome(
7901    event: &HistoryEvent,
7902    fallback_codec: &str,
7903    recorded_activity_type: Option<String>,
7904) -> Result<ActivityOutcome> {
7905    if event.event_type == "ActivityCompleted" {
7906        let codec = event
7907            .payload
7908            .get("payload_codec")
7909            .and_then(Value::as_str)
7910            .unwrap_or(fallback_codec);
7911        return Ok(Ok(decode_wire_avro_value(
7912            event.payload.get("result").unwrap_or(&Value::Null),
7913            codec,
7914        )?));
7915    }
7916
7917    let payload = &event.payload;
7918    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7919        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7920        "ActivityCancelled" => (
7921            ActivityFailureKind::Cancelled,
7922            "cancelled",
7923            "activity was cancelled",
7924        ),
7925        "ActivityTimedOut" => (
7926            ActivityFailureKind::TimedOut,
7927            "timeout",
7928            "activity timed out",
7929        ),
7930        _ => unreachable!("activity_outcome is called only for terminal activity events"),
7931    };
7932    let exception = payload
7933        .get("exception")
7934        .filter(|value| !value.is_null())
7935        .cloned();
7936    let failure_category = payload_string(payload, "failure_category");
7937    let timeout_kind = payload_string(payload, "timeout_kind");
7938    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7939        ActivityFailureKind::Failed => failure_category
7940            .clone()
7941            .unwrap_or_else(|| fallback_reason.to_string()),
7942        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7943        ActivityFailureKind::TimedOut => timeout_kind
7944            .clone()
7945            .unwrap_or_else(|| fallback_reason.to_string()),
7946    });
7947    let message = payload_string(payload, "message")
7948        .or_else(|| {
7949            exception
7950                .as_ref()
7951                .and_then(|value| payload_string(value, "message"))
7952        })
7953        .unwrap_or_else(|| fallback_message.to_string());
7954
7955    Ok(Err(ActivityFailure {
7956        kind,
7957        reason,
7958        message,
7959        activity_execution_id: payload_string(payload, "activity_execution_id"),
7960        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7961        activity_type: payload_string(payload, "activity_type")
7962            .or_else(|| payload_string(payload, "activity_name"))
7963            .or(recorded_activity_type),
7964        activity_class: payload_string(payload, "activity_class"),
7965        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7966        failure_id: payload_string(payload, "failure_id"),
7967        failure_category,
7968        timeout_kind,
7969        non_retryable: payload
7970            .get("non_retryable")
7971            .and_then(Value::as_bool)
7972            .unwrap_or(false),
7973        exception_type: payload_string(payload, "exception_type").or_else(|| {
7974            exception
7975                .as_ref()
7976                .and_then(|value| payload_string(value, "type"))
7977        }),
7978        exception_class: payload_string(payload, "exception_class").or_else(|| {
7979            exception
7980                .as_ref()
7981                .and_then(|value| payload_string(value, "class"))
7982        }),
7983        code: payload
7984            .get("code")
7985            .filter(|value| !value.is_null())
7986            .cloned(),
7987        exception,
7988    }))
7989}
7990
7991type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7992
7993fn child_workflow_outcomes(
7994    events: &[HistoryEvent],
7995    fallback_codec: &str,
7996    parent: WorkflowIdentity,
7997) -> Result<Vec<ChildWorkflowOutcome>> {
7998    let mut outcomes = Vec::new();
7999
8000    for event in events {
8001        let kind = match event.event_type.as_str() {
8002            "ChildRunCompleted" => None,
8003            "ChildRunFailed" => Some((
8004                ChildWorkflowFailureKind::Failed,
8005                "child_workflow",
8006                "child workflow failed",
8007            )),
8008            "ChildRunCancelled" => Some((
8009                ChildWorkflowFailureKind::Cancelled,
8010                "cancelled",
8011                "child workflow was cancelled",
8012            )),
8013            "ChildRunTerminated" => Some((
8014                ChildWorkflowFailureKind::Terminated,
8015                "terminated",
8016                "child workflow was terminated",
8017            )),
8018            _ => continue,
8019        };
8020        let payload = &event.payload;
8021        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
8022        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
8023        let child_workflow_type = payload_string(payload, "child_workflow_type");
8024
8025        if let Some((kind, reason, fallback_message)) = kind {
8026            let exception = payload
8027                .get("exception")
8028                .filter(|value| !value.is_null())
8029                .cloned();
8030            let message = payload_string(payload, "message")
8031                .or_else(|| {
8032                    exception
8033                        .as_ref()
8034                        .and_then(|value| payload_string(value, "message"))
8035                })
8036                .unwrap_or_else(|| fallback_message.to_string());
8037            let exception_type = payload_string(payload, "exception_type").or_else(|| {
8038                exception
8039                    .as_ref()
8040                    .and_then(|value| payload_string(value, "type"))
8041            });
8042            let exception_class = payload_string(payload, "exception_class").or_else(|| {
8043                exception
8044                    .as_ref()
8045                    .and_then(|value| payload_string(value, "class"))
8046            });
8047            outcomes.push(Err(ChildWorkflowFailure {
8048                kind,
8049                reason: reason.to_string(),
8050                message,
8051                parent_workflow_id: parent.workflow_id.clone(),
8052                parent_workflow_run_id: parent.run_id.clone(),
8053                child_workflow_id,
8054                child_workflow_run_id,
8055                child_workflow_type,
8056                failure_id: payload_string(payload, "failure_id"),
8057                failure_category: payload_string(payload, "failure_category"),
8058                exception_type,
8059                exception_class,
8060                non_retryable: payload
8061                    .get("non_retryable")
8062                    .and_then(Value::as_bool)
8063                    .unwrap_or(false),
8064                code: payload
8065                    .get("code")
8066                    .filter(|value| !value.is_null())
8067                    .cloned(),
8068                exception,
8069            }));
8070            continue;
8071        }
8072
8073        let codec = payload
8074            .get("payload_codec")
8075            .and_then(Value::as_str)
8076            .unwrap_or(fallback_codec);
8077        let result = payload
8078            .get("result")
8079            .or_else(|| payload.get("output"))
8080            .unwrap_or(&Value::Null);
8081        outcomes.push(Ok(ChildWorkflowAvroResult {
8082            parent: parent.clone(),
8083            child: WorkflowIdentity {
8084                workflow_id: child_workflow_id,
8085                run_id: child_workflow_run_id,
8086            },
8087            child_workflow_type,
8088            result: decode_wire_avro_value(result, codec)?,
8089        }));
8090    }
8091
8092    Ok(outcomes)
8093}
8094
8095fn payload_string(payload: &Value, key: &str) -> Option<String> {
8096    payload
8097        .get(key)
8098        .and_then(Value::as_str)
8099        .filter(|value| !value.is_empty())
8100        .map(str::to_string)
8101}
8102
8103fn workflow_failure_command(error: &Error) -> Value {
8104    let (exception_type, exception_class, properties) = match error {
8105        Error::ActivityFailed(failure) => (
8106            match failure.kind {
8107                ActivityFailureKind::Failed => "ActivityFailed",
8108                ActivityFailureKind::Cancelled => "ActivityCancelled",
8109                ActivityFailureKind::TimedOut => "ActivityTimedOut",
8110            },
8111            "durable_workflow::ActivityFailure",
8112            json!({
8113                "reason": failure.reason,
8114                "activity_execution_id": failure.activity_execution_id,
8115                "activity_attempt_id": failure.activity_attempt_id,
8116                "activity_type": failure.activity_type,
8117                "activity_class": failure.activity_class,
8118                "attempt_number": failure.attempt_number,
8119                "failure_id": failure.failure_id,
8120                "failure_category": failure.failure_category,
8121                "timeout_kind": failure.timeout_kind,
8122                "activity_non_retryable": failure.non_retryable,
8123                "activity_exception_type": failure.exception_type,
8124                "activity_exception_class": failure.exception_class,
8125                "activity_code": failure.code,
8126                "activity_exception": failure.exception,
8127            }),
8128        ),
8129        Error::ChildWorkflowFailed(failure) => (
8130            match failure.kind {
8131                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
8132                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
8133                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
8134            },
8135            "durable_workflow::ChildWorkflowFailure",
8136            json!({
8137                "reason": failure.reason,
8138                "parent_workflow_id": failure.parent_workflow_id,
8139                "parent_workflow_run_id": failure.parent_workflow_run_id,
8140                "child_workflow_id": failure.child_workflow_id,
8141                "child_workflow_run_id": failure.child_workflow_run_id,
8142                "child_workflow_type": failure.child_workflow_type,
8143                "failure_id": failure.failure_id,
8144                "failure_category": failure.failure_category,
8145                "child_exception_type": failure.exception_type,
8146                "child_exception_class": failure.exception_class,
8147                "child_non_retryable": failure.non_retryable,
8148                "child_code": failure.code,
8149                "child_exception": failure.exception,
8150            }),
8151        ),
8152        Error::NonDeterministicReplay(_) => (
8153            "NonDeterministicReplay",
8154            "durable_workflow::Error",
8155            Value::Null,
8156        ),
8157        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
8158    };
8159    let non_retryable = match error {
8160        Error::ActivityFailed(failure) => failure.non_retryable,
8161        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
8162        Error::NonDeterministicReplay(_) => true,
8163        _ => false,
8164    };
8165
8166    json!({
8167        "type": "fail_workflow",
8168        "message": error.to_string(),
8169        "exception_type": exception_type,
8170        "exception_class": exception_class,
8171        "non_retryable": non_retryable,
8172        "exception": {
8173            "type": exception_type,
8174            "class": exception_class,
8175            "message": error.to_string(),
8176            "properties": properties,
8177        }
8178    })
8179}
8180
8181fn workflow_task_integrity_error(error: &Error) -> bool {
8182    matches!(
8183        error,
8184        Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
8185    )
8186}
8187
8188fn decode_signal_event_arguments(
8189    event: &HistoryEvent,
8190    fallback_codec: &str,
8191) -> Result<Vec<AvroValue>> {
8192    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
8193    validate_payload_codec(codec)?;
8194    let raw = signal_history_payload(&event.payload);
8195    let decoded = match raw.filter(|value| !value.is_null()) {
8196        Some(value) => decode_wire_avro_value(value, codec)?,
8197        None => AvroValue::Array(Vec::new()),
8198    };
8199    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
8200        unreachable!("normalize_avro_arguments always returns an array");
8201    };
8202    Ok(arguments)
8203}
8204
8205fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8206    let Some(export_events) = task
8207        .history_export
8208        .as_ref()
8209        .and_then(|export| export.get("history_events"))
8210        .and_then(Value::as_array)
8211    else {
8212        return Ok(());
8213    };
8214
8215    if export_events.len() > task.history_events.len() {
8216        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8217    }
8218
8219    Ok(())
8220}
8221
8222fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8223    let Some(export) = task.history_export.as_ref() else {
8224        return Ok(());
8225    };
8226    let signals = export
8227        .get("signals")
8228        .and_then(Value::as_array)
8229        .cloned()
8230        .unwrap_or_default();
8231    let activities = export
8232        .get("activities")
8233        .and_then(Value::as_array)
8234        .cloned()
8235        .unwrap_or_default();
8236    let export_codec = export
8237        .get("payloads")
8238        .and_then(|payloads| payloads.get("codec"))
8239        .and_then(Value::as_str)
8240        .unwrap_or(&task.payload_codec)
8241        .to_string();
8242    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8243
8244    for event in &mut task.history_events {
8245        if event.event_type == "ActivityCompleted" {
8246            let sequence = event
8247                .payload
8248                .get("sequence")
8249                .or_else(|| event.payload.get("workflow_sequence"))
8250                .and_then(value_as_u64);
8251            let Some(activity) = sequence.and_then(|sequence| {
8252                activities.iter().find(|activity| {
8253                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8254                })
8255            }) else {
8256                continue;
8257            };
8258            let Some(payload) = event.payload.as_object_mut() else {
8259                continue;
8260            };
8261            if missing_payload(payload.get("result")) {
8262                if let Some(result) = activity
8263                    .get("result")
8264                    .filter(|value| !missing_payload(Some(value)))
8265                {
8266                    payload.insert("result".to_string(), result.clone());
8267                }
8268            }
8269            for field in ["payload_codec", "activity_type"] {
8270                if payload
8271                    .get(field)
8272                    .and_then(Value::as_str)
8273                    .unwrap_or_default()
8274                    .is_empty()
8275                {
8276                    if let Some(value) = activity.get(field) {
8277                        payload.insert(field.to_string(), value.clone());
8278                    }
8279                }
8280            }
8281            continue;
8282        }
8283
8284        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8285            continue;
8286        }
8287        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8288        let command_id = event
8289            .payload
8290            .get("workflow_command_id")
8291            .or_else(|| event.raw.get("workflow_command_id"))
8292            .and_then(Value::as_str);
8293        let signal_name = event
8294            .payload
8295            .get("signal_name")
8296            .and_then(Value::as_str)
8297            .unwrap_or_default()
8298            .to_string();
8299        let matched = signals
8300            .iter()
8301            .find(|signal| {
8302                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8303            })
8304            .or_else(|| {
8305                signals.iter().find(|signal| {
8306                    command_id.is_some()
8307                        && signal.get("command_id").and_then(Value::as_str) == command_id
8308                })
8309            })
8310            .or_else(|| {
8311                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8312                let signal = signals
8313                    .iter()
8314                    .filter(|signal| {
8315                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8316                    })
8317                    .nth(*offset);
8318                if signal.is_some() {
8319                    *offset += 1;
8320                }
8321                signal
8322            });
8323        let Some(signal) = matched else {
8324            continue;
8325        };
8326        let signal_codec = signal
8327            .get("payload_codec")
8328            .and_then(Value::as_str)
8329            .unwrap_or(&export_codec);
8330        let Some(payload) = event.payload.as_object_mut() else {
8331            continue;
8332        };
8333        if missing_payload(payload.get("arguments")) {
8334            if let Some(arguments) = signal
8335                .get("arguments")
8336                .filter(|value| !missing_payload(Some(value)))
8337            {
8338                let envelope = match arguments {
8339                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8340                    other => other.clone(),
8341                };
8342                payload.insert("arguments".to_string(), envelope);
8343            }
8344        }
8345        if payload
8346            .get("payload_codec")
8347            .and_then(Value::as_str)
8348            .unwrap_or_default()
8349            .is_empty()
8350        {
8351            payload.insert("payload_codec".to_string(), json!(signal_codec));
8352        }
8353    }
8354
8355    Ok(())
8356}
8357
8358fn missing_payload(value: Option<&Value>) -> bool {
8359    match value {
8360        None | Some(Value::Null) => true,
8361        Some(Value::String(value)) => value.is_empty(),
8362        Some(_) => false,
8363    }
8364}
8365
8366fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8367    let export_signals = task
8368        .history_export
8369        .as_ref()
8370        .and_then(|export| export.get("signals"))
8371        .and_then(Value::as_array)
8372        .cloned()
8373        .unwrap_or_default();
8374    let export_codec = task
8375        .history_export
8376        .as_ref()
8377        .and_then(|export| export.get("payloads"))
8378        .and_then(|payloads| payloads.get("codec"))
8379        .and_then(Value::as_str)
8380        .unwrap_or(&task.payload_codec);
8381    let mut name_offsets: HashMap<String, usize> = HashMap::new();
8382    let mut signals = Vec::new();
8383
8384    for event in &task.history_events {
8385        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8386            continue;
8387        }
8388
8389        let name = event
8390            .payload
8391            .get("signal_name")
8392            .and_then(Value::as_str)
8393            .unwrap_or_default();
8394        if name.is_empty() {
8395            continue;
8396        }
8397        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8398        let command_id = event
8399            .payload
8400            .get("workflow_command_id")
8401            .or_else(|| event.raw.get("workflow_command_id"))
8402            .and_then(Value::as_str);
8403        let matched_export = export_signals
8404            .iter()
8405            .find(|candidate| {
8406                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8407            })
8408            .or_else(|| {
8409                export_signals.iter().find(|candidate| {
8410                    command_id.is_some()
8411                        && candidate.get("command_id").and_then(Value::as_str) == command_id
8412                })
8413            })
8414            .or_else(|| {
8415                let offset = name_offsets.entry(name.to_string()).or_default();
8416                let candidate = export_signals
8417                    .iter()
8418                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8419                    .nth(*offset);
8420                if candidate.is_some() {
8421                    *offset += 1;
8422                }
8423                candidate
8424            });
8425        let codec = event
8426            .payload
8427            .get("payload_codec")
8428            .and_then(Value::as_str)
8429            .or_else(|| {
8430                matched_export
8431                    .and_then(|signal| signal.get("payload_codec"))
8432                    .and_then(Value::as_str)
8433            })
8434            .unwrap_or(export_codec);
8435        let raw_arguments = signal_history_payload(&event.payload)
8436            .filter(|value| !value.is_null())
8437            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8438        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8439        let workflow_sequence = event
8440            .payload
8441            .get("workflow_sequence")
8442            .and_then(value_as_u64)
8443            .or_else(|| {
8444                matched_export
8445                    .and_then(|signal| signal.get("workflow_sequence"))
8446                    .and_then(value_as_u64)
8447            });
8448
8449        signals.push(QuerySignal {
8450            id: signal_id.map(str::to_string).or_else(|| {
8451                matched_export
8452                    .and_then(|signal| signal.get("id"))
8453                    .and_then(Value::as_str)
8454                    .map(str::to_string)
8455            }),
8456            name: name.to_string(),
8457            arguments,
8458            avro_arguments,
8459            workflow_sequence,
8460        });
8461    }
8462
8463    if signals.is_empty() {
8464        for signal in export_signals {
8465            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8466                continue;
8467            }
8468            let Some(name) = signal.get("name").and_then(Value::as_str) else {
8469                continue;
8470            };
8471            let codec = signal
8472                .get("payload_codec")
8473                .and_then(Value::as_str)
8474                .unwrap_or(export_codec);
8475            let (arguments, avro_arguments) =
8476                decode_query_signal_arguments(signal.get("arguments"), codec)?;
8477            signals.push(QuerySignal {
8478                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8479                name: name.to_string(),
8480                arguments,
8481                avro_arguments,
8482                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8483            });
8484        }
8485        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8486    }
8487
8488    Ok(signals)
8489}
8490
8491fn decode_query_signal_arguments(
8492    raw: Option<&Value>,
8493    codec: &str,
8494) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8495    validate_payload_codec(codec)?;
8496    let decoded = match raw.filter(|value| !value.is_null()) {
8497        Some(value) => decode_wire_avro_value(value, codec)?,
8498        None => AvroValue::Array(Vec::new()),
8499    };
8500    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8501        unreachable!("normalize_avro_arguments always returns an array");
8502    };
8503    let arguments = avro_arguments
8504        .iter()
8505        .cloned()
8506        .map(AvroValue::into_json)
8507        .collect::<Result<Vec<_>>>()?;
8508    Ok((arguments, avro_arguments))
8509}
8510
8511fn value_as_u64(value: &Value) -> Option<u64> {
8512    value
8513        .as_u64()
8514        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8515}
8516
8517#[cfg(test)]
8518mod tests {
8519    use super::*;
8520    use std::{
8521        io::{Read, Write},
8522        net::{SocketAddr, TcpListener, TcpStream},
8523        sync::atomic::AtomicUsize,
8524        thread,
8525    };
8526
8527    fn fixture_envelope(value: Value) -> Value {
8528        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
8529    }
8530
8531    fn fixture_blob(value: Value) -> String {
8532        encode_payload(&value, DEFAULT_CODEC)
8533            .expect("encode Avro test fixture")
8534            .blob
8535    }
8536
8537    #[test]
8538    fn client_builder_rejects_the_sdk_owned_api_suffix() {
8539        for base_url in [
8540            "http://127.0.0.1:8080/api",
8541            "http://localhost:8080/api/",
8542            "https://runtime.example.test/namespaces/orders/api",
8543        ] {
8544            let error = Client::builder(base_url)
8545                .build()
8546                .expect_err("SDK-owned /api suffix must be rejected during build");
8547
8548            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8549            assert!(
8550                error.to_string().contains("SDK appends /api automatically"),
8551                "the validation error must explain how to fix the endpoint"
8552            );
8553        }
8554    }
8555
8556    #[test]
8557    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8558        for (base_url, expected) in [
8559            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8560            (
8561                "http://localhost:8080/durable-workflow/",
8562                "http://localhost:8080/durable-workflow",
8563            ),
8564            (
8565                "https://runtime.example.test/namespaces/orders",
8566                "https://runtime.example.test/namespaces/orders",
8567            ),
8568            (
8569                "https://runtime.example.test/gateway/api/namespaces/orders",
8570                "https://runtime.example.test/gateway/api/namespaces/orders",
8571            ),
8572            (
8573                "https://api.example.test/runtime/orders/",
8574                "https://api.example.test/runtime/orders",
8575            ),
8576        ] {
8577            let client = Client::builder(base_url)
8578                .build()
8579                .expect("Server and Cloud runtime base URL must remain valid");
8580
8581            assert_eq!(client.base_url, expected);
8582        }
8583    }
8584
8585    fn typed_fidelity_probe() -> AvroValue {
8586        AvroValue::Map(BTreeMap::from([
8587            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8588            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8589            (
8590                "numeric".to_string(),
8591                AvroValue::Map(BTreeMap::from([
8592                    ("0".to_string(), AvroValue::String("zero".to_string())),
8593                    ("1".to_string(), AvroValue::String("one".to_string())),
8594                ])),
8595            ),
8596            (
8597                "nested".to_string(),
8598                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8599                    "enabled".to_string(),
8600                    AvroValue::Boolean(true),
8601                )]))]),
8602            ),
8603            (
8604                "projection_collisions".to_string(),
8605                AvroValue::Array(projection_collision_probe()),
8606            ),
8607        ]))
8608    }
8609
8610    fn projection_collision_probe() -> Vec<AvroValue> {
8611        vec![
8612            AvroValue::Map(BTreeMap::from([
8613                ("$type".to_string(), AvroValue::String("bytes".to_string())),
8614                (
8615                    "base64".to_string(),
8616                    AvroValue::String("ordinary user text".to_string()),
8617                ),
8618            ])),
8619            AvroValue::Map(BTreeMap::from([
8620                ("$type".to_string(), AvroValue::String("map".to_string())),
8621                (
8622                    "entries".to_string(),
8623                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8624                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
8625                        (
8626                            "value".to_string(),
8627                            AvroValue::String("user map".to_string()),
8628                        ),
8629                    ]))]),
8630                ),
8631            ])),
8632        ]
8633    }
8634
8635    #[derive(Clone, Debug, Default, PartialEq)]
8636    struct ReplayCounterState {
8637        loaded: Option<String>,
8638        count: i64,
8639        finished: bool,
8640    }
8641
8642    fn replay_counter_worker() -> Worker {
8643        let client = Client::new("http://127.0.0.1:8080").expect("client");
8644        let mut worker = Worker::new(client, "rust-workers");
8645        worker.register_replayed_workflow(
8646            "replay-counter",
8647            ReplayCounterState::default,
8648            |ctx, _input, state| async move {
8649                let loaded = ctx.activity("load-counter", json!([])).await?;
8650                state.update(|current| {
8651                    current.loaded = loaded.as_str().map(str::to_string);
8652                })?;
8653                for _ in 0..2 {
8654                    let signal = ctx.wait_signal("increment").await?;
8655                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8656                    state.update(|current| current.count += amount)?;
8657                }
8658                state.update(|current| current.finished = true)?;
8659                state.read(|current| Ok(json!(current.count)))?
8660            },
8661        );
8662        worker.register_replayed_query::<ReplayCounterState, _, _>(
8663            "replay-counter",
8664            "current",
8665            |_ctx, state, _args| async move {
8666                Ok(json!({
8667                    "loaded": state.loaded,
8668                    "count": state.count,
8669                    "finished": state.finished,
8670                }))
8671            },
8672        );
8673        worker.register_replayed_query::<ReplayCounterState, _, _>(
8674            "replay-counter",
8675            "detached-mutation",
8676            |_ctx, state, _args| async move {
8677                let mut detached = (*state).clone();
8678                detached.count = 999;
8679                Ok(json!(detached.count))
8680            },
8681        );
8682        worker.register_replayed_query::<ReplayCounterState, _, _>(
8683            "replay-counter",
8684            "failed-mutation",
8685            |_ctx, state, _args| async move {
8686                let mut detached = (*state).clone();
8687                detached.count = 999;
8688                Err(Error::WorkerLoop("query refused".to_string()))
8689            },
8690        );
8691        worker
8692    }
8693
8694    fn replay_counter_query(
8695        query_name: &str,
8696        history_events: Value,
8697        run_status: &str,
8698    ) -> QueryTask {
8699        let arguments = fixture_envelope(json!([]));
8700        serde_json::from_value(json!({
8701            "query_task_id": format!("query-{query_name}"),
8702            "workflow_type": "replay-counter",
8703            "query_name": query_name,
8704            "payload_codec": DEFAULT_CODEC,
8705            "workflow_arguments": arguments.clone(),
8706            "query_arguments": arguments,
8707            "history_events": history_events,
8708            "run_status": run_status,
8709        }))
8710        .expect("query task")
8711    }
8712
8713    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8714        workflow_context_with_codec(history, DEFAULT_CODEC)
8715    }
8716
8717    fn workflow_context_with_codec(
8718        history: Vec<HistoryEvent>,
8719        payload_codec: &str,
8720    ) -> WorkflowContext {
8721        WorkflowContext {
8722            state: Arc::new(Mutex::new(
8723                WorkflowState::new_with_identity(
8724                    history,
8725                    None,
8726                    None,
8727                    "rust-workers".to_string(),
8728                    payload_codec.to_string(),
8729                    None,
8730                )
8731                .expect("valid workflow history"),
8732            )),
8733        }
8734    }
8735
8736    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8737        HistoryEvent {
8738            event_type: event_type.to_string(),
8739            payload,
8740            raw: HashMap::new(),
8741        }
8742    }
8743
8744    fn workflow_task(
8745        workflow_type: &str,
8746        history_events: Vec<HistoryEvent>,
8747        payload_codec: &str,
8748    ) -> WorkflowTask {
8749        WorkflowTask {
8750            task_id: format!("wft-{workflow_type}"),
8751            workflow_id: Some(format!("wf-{workflow_type}")),
8752            run_id: Some(format!("run-{workflow_type}")),
8753            workflow_type: workflow_type.to_string(),
8754            payload_codec: payload_codec.to_string(),
8755            arguments: Some(
8756                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8757            ),
8758            total_history_events: Some(history_events.len() as u64),
8759            history_size_bytes: None,
8760            continue_as_new_recommended: None,
8761            history_budget_pressure: None,
8762            history_events,
8763            next_history_page_token: None,
8764            workflow_task_attempt: 1,
8765            workflow_signal_id: None,
8766            signal_name: None,
8767            signal_arguments: None,
8768            workflow_update_id: None,
8769            update_name: None,
8770            lease_owner: Some("rust-worker".to_string()),
8771        }
8772    }
8773
8774    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8775    struct SideEffectProbe {
8776        request_id: String,
8777        attempt: u32,
8778    }
8779
8780    #[test]
8781    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8782        let calls = AtomicUsize::new(0);
8783        let ctx = workflow_context(Vec::new());
8784        let value = ctx
8785            .side_effect(|| {
8786                calls.fetch_add(1, Ordering::SeqCst);
8787                SideEffectProbe {
8788                    request_id: "request-42".to_string(),
8789                    attempt: 3,
8790                }
8791            })
8792            .expect("first side effect");
8793        assert_eq!(value.attempt, 3);
8794        assert_eq!(calls.load(Ordering::SeqCst), 1);
8795        let commands = ctx.take_commands().expect("commands");
8796        assert_eq!(commands.len(), 1);
8797        assert_eq!(commands[0]["type"], "record_side_effect");
8798        assert_eq!(
8799            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8800            serde_json::to_value(&value).expect("value")
8801        );
8802
8803        let replay = workflow_context(vec![history_event(
8804            "SideEffectRecorded",
8805            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8806        )]);
8807        let replayed: SideEffectProbe = replay
8808            .side_effect(|| {
8809                calls.fetch_add(1, Ordering::SeqCst);
8810                panic!("committed side-effect callbacks must not run during replay")
8811            })
8812            .expect("replayed side effect");
8813        assert_eq!(replayed, value);
8814        assert_eq!(calls.load(Ordering::SeqCst), 1);
8815        assert!(replay.take_commands().expect("commands").is_empty());
8816        replay.ensure_history_consumed().expect("history consumed");
8817    }
8818
8819    #[test]
8820    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8821        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8822        let value = ctx
8823            .side_effect(|| SideEffectProbe {
8824                request_id: "avro-request".to_string(),
8825                attempt: 1,
8826            })
8827            .expect("Avro side effect");
8828        let uuid = ctx.uuid_v4().expect("deterministic UUID");
8829        let commands = ctx.take_commands().expect("commands");
8830        assert_eq!(commands.len(), 2);
8831        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8832        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8833        assert_eq!(
8834            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8835            serde_json::to_value(&value).expect("value")
8836        );
8837
8838        let replay = workflow_context_with_codec(
8839            vec![
8840                history_event(
8841                    "SideEffectRecorded",
8842                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8843                ),
8844                history_event(
8845                    "SideEffectRecorded",
8846                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8847                ),
8848            ],
8849            DEFAULT_CODEC,
8850        );
8851        let replayed: SideEffectProbe = replay
8852            .side_effect(|| panic!("Avro callback must not run"))
8853            .expect("replayed Avro value");
8854        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8855        assert_eq!(replayed, value);
8856        assert_eq!(replayed_uuid, uuid);
8857        assert!(replay.take_commands().expect("commands").is_empty());
8858    }
8859
8860    #[test]
8861    fn typed_side_effect_replay_preserves_bytes_and_maps() {
8862        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8863        let value = ctx
8864            .side_effect_avro_value(typed_fidelity_probe)
8865            .expect("typed side effect");
8866        let commands = ctx.take_commands().expect("side-effect command");
8867        assert_eq!(
8868            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8869                .expect("recorded side effect"),
8870            value
8871        );
8872
8873        let replay = workflow_context_with_codec(
8874            vec![history_event(
8875                "SideEffectRecorded",
8876                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8877            )],
8878            DEFAULT_CODEC,
8879        );
8880        assert_eq!(
8881            replay
8882                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8883                .expect("replayed typed side effect"),
8884            value
8885        );
8886    }
8887
8888    #[test]
8889    fn ordered_side_effects_share_the_durable_command_stream() {
8890        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
8891        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
8892        let ctx = workflow_context(vec![
8893            history_event(
8894                "SideEffectRecorded",
8895                json!({"sequence": 1, "result": first}),
8896            ),
8897            history_event(
8898                "SideEffectRecorded",
8899                json!({"sequence": 2, "result": second}),
8900            ),
8901        ]);
8902        let first: String = ctx
8903            .side_effect(|| panic!("first callback must not run"))
8904            .expect("first replay");
8905        let second: i32 = ctx
8906            .side_effect(|| panic!("second callback must not run"))
8907            .expect("second replay");
8908        assert_eq!(first, "first");
8909        assert_eq!(second, 29);
8910        ctx.ensure_history_consumed().expect("ordered history");
8911
8912        let reordered = workflow_context(vec![history_event(
8913            "VersionMarkerRecorded",
8914            json!({
8915                "sequence": 1,
8916                "change_id": "before-side-effect",
8917                "version": 1,
8918                "min_supported": 1,
8919                "max_supported": 1,
8920            }),
8921        )]);
8922        let error = reordered
8923            .side_effect(|| "new".to_string())
8924            .expect_err("command reordering must fail");
8925        assert!(matches!(
8926            error,
8927            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8928                if reason == "recorded_command_mismatch"
8929        ));
8930    }
8931
8932    #[test]
8933    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8934        let ctx = workflow_context(Vec::new());
8935        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8936        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8937        assert!(ctx.patched("new-search").expect("patch"));
8938        ctx.deprecate_patch("new-search").expect("deprecate patch");
8939        let commands = ctx.take_commands().expect("commands");
8940        assert_eq!(commands.len(), 2);
8941        assert_eq!(commands[0]["type"], "record_version_marker");
8942        assert_eq!(commands[0]["version"], 2);
8943        assert_eq!(commands[1]["change_id"], "new-search");
8944
8945        let replay = workflow_context(vec![history_event(
8946            "VersionMarkerRecorded",
8947            json!({
8948                "sequence": 1,
8949                "change_id": "checkout-v2",
8950                "version": 2,
8951                "min_supported": 1,
8952                "max_supported": 2,
8953            }),
8954        )]);
8955        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8956        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8957        assert!(replay.take_commands().expect("commands").is_empty());
8958        replay.ensure_history_consumed().expect("history consumed");
8959    }
8960
8961    #[test]
8962    fn version_markers_reject_incompatible_or_malformed_history() {
8963        let incompatible = workflow_context(vec![history_event(
8964            "VersionMarkerRecorded",
8965            json!({
8966                "sequence": 1,
8967                "change_id": "checkout-v2",
8968                "version": 1,
8969                "min_supported": 1,
8970                "max_supported": 2,
8971            }),
8972        )]);
8973        let error = incompatible
8974            .get_version("checkout-v2", 2, 3)
8975            .expect_err("old version is unsupported");
8976        assert!(matches!(
8977            error,
8978            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8979                if reason == "version_marker_incompatible_range"
8980        ));
8981
8982        for (history, reason) in [
8983            (
8984                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8985                "side_effect_result_missing",
8986            ),
8987            (
8988                vec![history_event(
8989                    "SideEffectRecorded",
8990                    json!({
8991                        "sequence": 1,
8992                        "result": {"codec": "avro", "blob": "not-base64"},
8993                    }),
8994                )],
8995                "side_effect_payload_incompatible",
8996            ),
8997            (
8998                vec![history_event(
8999                    "SideEffectRecorded",
9000                    json!({"sequence": 1, "result": {"unwrapped": true}}),
9001                )],
9002                "side_effect_payload_malformed",
9003            ),
9004            (
9005                vec![history_event(
9006                    "VersionMarkerRecorded",
9007                    json!({
9008                        "sequence": 1,
9009                        "change_id": "change",
9010                        "version": 1,
9011                        "min_supported": 2,
9012                        "max_supported": 1,
9013                    }),
9014                )],
9015                "version_marker_history_range_invalid",
9016            ),
9017        ] {
9018            let error = WorkflowState::new(
9019                history,
9020                "rust-workers".to_string(),
9021                DEFAULT_CODEC.to_string(),
9022                None,
9023            )
9024            .expect_err("malformed history must fail");
9025            assert!(matches!(
9026                error,
9027                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
9028                    if actual == reason
9029            ));
9030        }
9031    }
9032
9033    #[test]
9034    fn duplicate_side_effects_and_version_markers_are_rejected() {
9035        let duplicate_side_effect = WorkflowState::new(
9036            vec![
9037                history_event(
9038                    "SideEffectRecorded",
9039                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
9040                ),
9041                history_event(
9042                    "SideEffectRecorded",
9043                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
9044                ),
9045            ],
9046            "rust-workers".to_string(),
9047            DEFAULT_CODEC.to_string(),
9048            None,
9049        )
9050        .expect_err("duplicate side effect");
9051        assert!(matches!(
9052            duplicate_side_effect,
9053            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9054                if reason == "duplicate_side_effect_record"
9055        ));
9056
9057        let marker = |sequence| {
9058            history_event(
9059                "VersionMarkerRecorded",
9060                json!({
9061                    "sequence": sequence,
9062                    "change_id": "same-change",
9063                    "version": 1,
9064                    "min_supported": 1,
9065                    "max_supported": 1,
9066                }),
9067            )
9068        };
9069        let duplicate_marker = WorkflowState::new(
9070            vec![marker(1), marker(3)],
9071            "rust-workers".to_string(),
9072            DEFAULT_CODEC.to_string(),
9073            None,
9074        )
9075        .expect_err("duplicate marker");
9076        assert!(matches!(
9077            duplicate_marker,
9078            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9079                if reason == "duplicate_version_marker"
9080        ));
9081    }
9082
9083    #[test]
9084    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
9085        fn worker(calls: Arc<AtomicUsize>) -> Worker {
9086            let client = Client::new("http://127.0.0.1:8080").expect("client");
9087            let mut worker = Worker::new(client, "rust-workers");
9088            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
9089                let calls = Arc::clone(&calls);
9090                async move {
9091                    let captured = ctx.side_effect(|| {
9092                        calls.fetch_add(1, Ordering::SeqCst);
9093                        "captured-once".to_string()
9094                    })?;
9095                    let version = ctx.get_version("cold-restart", 1, 2)?;
9096                    Ok(json!({"captured": captured, "version": version}))
9097                }
9098            });
9099            worker
9100        }
9101
9102        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
9103            WorkflowTask {
9104                task_id: "wft-side-effect-version".to_string(),
9105                workflow_id: Some("wf-side-effect-version".to_string()),
9106                run_id: Some("run-side-effect-version".to_string()),
9107                workflow_type: "rust.side-effect-version".to_string(),
9108                payload_codec: DEFAULT_CODEC.to_string(),
9109                arguments: Some(
9110                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
9111                ),
9112                history_events,
9113                total_history_events: None,
9114                history_size_bytes: None,
9115                continue_as_new_recommended: None,
9116                history_budget_pressure: None,
9117                next_history_page_token: None,
9118                workflow_task_attempt: 1,
9119                workflow_signal_id: None,
9120                signal_name: None,
9121                signal_arguments: None,
9122                workflow_update_id: None,
9123                update_name: None,
9124                lease_owner: Some("rust-worker".to_string()),
9125            }
9126        }
9127
9128        let calls = Arc::new(AtomicUsize::new(0));
9129        let initial = worker(Arc::clone(&calls))
9130            .execute_workflow_task(task(Vec::new()))
9131            .expect("initial execution");
9132        assert_eq!(
9133            initial
9134                .iter()
9135                .map(|command| &command["type"])
9136                .collect::<Vec<_>>(),
9137            vec![
9138                "record_side_effect",
9139                "record_version_marker",
9140                "complete_workflow"
9141            ]
9142        );
9143        assert_eq!(calls.load(Ordering::SeqCst), 1);
9144
9145        let restarted = worker(Arc::clone(&calls));
9146        let replayed = restarted
9147            .execute_workflow_task(task(vec![
9148                history_event(
9149                    "SideEffectRecorded",
9150                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
9151                ),
9152                history_event(
9153                    "VersionMarkerRecorded",
9154                    json!({
9155                        "sequence": 2,
9156                        "change_id": "cold-restart",
9157                        "version": 2,
9158                        "min_supported": 1,
9159                        "max_supported": 2,
9160                    }),
9161                ),
9162            ]))
9163            .expect("cold replay");
9164        assert_eq!(replayed.len(), 1);
9165        assert_eq!(replayed[0]["type"], "complete_workflow");
9166        assert_eq!(calls.load(Ordering::SeqCst), 1);
9167    }
9168
9169    #[test]
9170    fn side_effect_replay_rejects_changed_rust_value_type() {
9171        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
9172        let ctx = workflow_context(vec![history_event(
9173            "SideEffectRecorded",
9174            json!({"sequence": 1, "result": result}),
9175        )]);
9176        let error = ctx
9177            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
9178            .expect_err("changed type must fail replay");
9179        assert!(matches!(
9180            error,
9181            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9182                if reason == "side_effect_type_mismatch"
9183        ));
9184    }
9185
9186    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
9187        vec![
9188            history_event(
9189                "ActivityScheduled",
9190                json!({
9191                    "sequence": 1,
9192                    "activity_type": "flaky",
9193                    "activity_execution_id": "act-1",
9194                    "activity": {
9195                        "id": "act-1",
9196                        "sequence": 1,
9197                        "type": "flaky",
9198                        "queue": "critical-activities",
9199                        "execution_mode": null,
9200                        "retry_policy": {
9201                            "snapshot_version": 1,
9202                            "max_attempts": 3,
9203                            "backoff_seconds": [2, 4],
9204                            "start_to_close_timeout": 30,
9205                            "schedule_to_start_timeout": 5,
9206                            "schedule_to_close_timeout": 90,
9207                            "heartbeat_timeout": 10,
9208                            "non_retryable_error_types": ["PermanentError"]
9209                        }
9210                    }
9211                }),
9212            ),
9213            history_event(
9214                "ActivityStarted",
9215                json!({
9216                    "sequence": 1,
9217                    "activity_type": "flaky",
9218                    "activity_execution_id": "act-1",
9219                    "activity_attempt_id": "attempt-1",
9220                    "attempt_number": 1
9221                }),
9222            ),
9223            history_event(
9224                "ActivityRetryScheduled",
9225                json!({
9226                    "sequence": 1,
9227                    "activity_type": "flaky",
9228                    "activity_execution_id": "act-1",
9229                    "activity_attempt_id": "attempt-1",
9230                    "attempt_number": 1,
9231                    "retry_after_attempt": 1,
9232                    "retry_backoff_seconds": 2,
9233                    "failure_category": "activity",
9234                    "exception_type": "TransientError"
9235                }),
9236            ),
9237            history_event(
9238                "ActivityStarted",
9239                json!({
9240                    "sequence": 1,
9241                    "activity_type": "flaky",
9242                    "activity_execution_id": "act-1",
9243                    "activity_attempt_id": "attempt-2",
9244                    "attempt_number": 2
9245                }),
9246            ),
9247            history_event(
9248                "ActivityCompleted",
9249                json!({
9250                    "sequence": 1,
9251                    "activity_type": "flaky",
9252                    "activity_execution_id": "act-1",
9253                    "activity_attempt_id": "attempt-2",
9254                    "attempt_number": 2,
9255                    "payload_codec": DEFAULT_CODEC,
9256                    "result": fixture_envelope(json!({"status":"recovered"}))
9257                }),
9258            ),
9259        ]
9260    }
9261
9262    fn retry_activity_options() -> ActivityOptions {
9263        ActivityOptions::new()
9264            .task_queue("critical-activities")
9265            .retry_policy(
9266                ActivityRetryPolicy::new(3)
9267                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9268                    .non_retryable_error_type("PermanentError"),
9269            )
9270            .start_to_close_timeout(Duration::from_secs(30))
9271            .schedule_to_start_timeout(Duration::from_secs(5))
9272            .schedule_to_close_timeout(Duration::from_secs(90))
9273            .heartbeat_timeout(Duration::from_secs(10))
9274    }
9275
9276    #[test]
9277    fn fixed_avro_value_round_trips_json_values() {
9278        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9279        let envelope = PayloadEnvelope::avro(&value).expect("encode");
9280        assert_eq!(envelope.codec, DEFAULT_CODEC);
9281        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9282    }
9283
9284    #[tokio::test]
9285    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9286        let client = Client::new("http://127.0.0.1:8080").expect("client");
9287        let mut worker = Worker::new(client, "rust-workers");
9288        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9289        worker
9290            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9291        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9292            Ok(input)
9293        });
9294        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9295            Ok(input)
9296        });
9297        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9298            Ok(AvroValue::Array(
9299                ctx.wait_signal_avro_value("changed").await?,
9300            ))
9301        });
9302
9303        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9304        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9305
9306        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9307        workflow.arguments = Some(envelope.clone());
9308        let commands = worker
9309            .execute_workflow_task(workflow)
9310            .expect("typed workflow task");
9311        assert_eq!(commands[0]["type"], "complete_workflow");
9312        assert_eq!(
9313            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9314                .expect("typed workflow result"),
9315            arguments
9316        );
9317
9318        let activity = ActivityTask {
9319            task_id: "activity-typed".to_string(),
9320            activity_attempt_id: Some("attempt-typed".to_string()),
9321            attempt_id: None,
9322            activity_type: "typed.activity".to_string(),
9323            payload_codec: DEFAULT_CODEC.to_string(),
9324            arguments: Some(envelope.clone()),
9325            attempt_number: 1,
9326            lease_owner: Some("rust-worker".to_string()),
9327        };
9328        assert_eq!(
9329            worker
9330                .execute_activity_task(activity)
9331                .await
9332                .expect("typed activity result"),
9333            arguments
9334        );
9335
9336        let query = QueryTask {
9337            query_task_id: "query-typed".to_string(),
9338            query_task_attempt: 1,
9339            lease_owner: Some("rust-worker".to_string()),
9340            workflow_id: Some("typed-1".to_string()),
9341            run_id: Some("run-typed".to_string()),
9342            workflow_type: "typed.echo".to_string(),
9343            query_name: "inspect".to_string(),
9344            payload_codec: DEFAULT_CODEC.to_string(),
9345            workflow_arguments: Some(
9346                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9347                    .expect("workflow input"),
9348            ),
9349            query_arguments: Some(envelope.clone()),
9350            history_events: Vec::new(),
9351            history_export: None,
9352            run_status: Some("running".to_string()),
9353        };
9354        assert_eq!(
9355            worker
9356                .execute_query_task(query)
9357                .await
9358                .expect("typed query result"),
9359            arguments
9360        );
9361
9362        let mut update = workflow_task(
9363            "typed.echo",
9364            vec![history_event(
9365                "UpdateAccepted",
9366                json!({
9367                    "update_id": "update-typed",
9368                    "update_name": "replace",
9369                    "arguments": envelope.clone(),
9370                }),
9371            )],
9372            DEFAULT_CODEC,
9373        );
9374        update.workflow_update_id = Some("update-typed".to_string());
9375        update.update_name = Some("replace".to_string());
9376        let commands = worker
9377            .execute_workflow_task(update)
9378            .expect("typed update task");
9379        assert_eq!(commands[0]["type"], "complete_update");
9380        assert_eq!(
9381            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9382                .expect("typed update result"),
9383            arguments
9384        );
9385
9386        let mut signal = workflow_task(
9387            "typed.signal",
9388            vec![history_event(
9389                "SignalReceived",
9390                json!({
9391                    "signal_id": "signal-typed",
9392                    "signal_name": "changed",
9393                    "arguments": envelope.clone(),
9394                }),
9395            )],
9396            DEFAULT_CODEC,
9397        );
9398        signal.workflow_signal_id = Some("signal-typed".to_string());
9399        signal.signal_name = Some("changed".to_string());
9400        signal.signal_arguments = Some(envelope);
9401        let commands = worker
9402            .execute_workflow_task(signal)
9403            .expect("typed signal resume");
9404        assert_eq!(
9405            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9406                .expect("typed signal result"),
9407            arguments
9408        );
9409    }
9410
9411    #[tokio::test]
9412    async fn typed_helpers_never_parse_json_inspection_projection() {
9413        let collision_values = projection_collision_probe();
9414        let expected = AvroValue::Array(collision_values.clone());
9415        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9416
9417        let activity_context = workflow_context_with_codec(
9418            vec![history_event(
9419                "ActivityCompleted",
9420                json!({
9421                    "sequence": 1,
9422                    "activity_type": "collision.activity",
9423                    "payload_codec": DEFAULT_CODEC,
9424                    "result": envelope.clone(),
9425                }),
9426            )],
9427            DEFAULT_CODEC,
9428        );
9429        assert_eq!(
9430            activity_context
9431                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9432                .await
9433                .expect("typed activity collision result"),
9434            expected
9435        );
9436
9437        let signal_context = workflow_context_with_codec(
9438            vec![
9439                history_event(
9440                    "SignalWaitOpened",
9441                    json!({"sequence": 1, "signal_name": "collision"}),
9442                ),
9443                history_event(
9444                    "SignalApplied",
9445                    json!({
9446                        "sequence": 1,
9447                        "signal_name": "collision",
9448                        "payload_codec": DEFAULT_CODEC,
9449                        "value": envelope.clone(),
9450                    }),
9451                ),
9452            ],
9453            DEFAULT_CODEC,
9454        );
9455        assert_eq!(
9456            signal_context
9457                .wait_signal_avro_value("collision")
9458                .await
9459                .expect("typed signal collision arguments"),
9460            collision_values
9461        );
9462
9463        let child_context = workflow_context_with_codec(
9464            vec![
9465                history_event(
9466                    "ChildWorkflowScheduled",
9467                    json!({
9468                        "sequence": 1,
9469                        "child_workflow_instance_id": "collision-child",
9470                        "child_workflow_run_id": "collision-run",
9471                        "child_workflow_type": "collision.child",
9472                    }),
9473                ),
9474                history_event(
9475                    "ChildRunCompleted",
9476                    json!({
9477                        "sequence": 1,
9478                        "child_workflow_instance_id": "collision-child",
9479                        "child_workflow_run_id": "collision-run",
9480                        "child_workflow_type": "collision.child",
9481                        "payload_codec": DEFAULT_CODEC,
9482                        "result": envelope,
9483                    }),
9484                ),
9485            ],
9486            DEFAULT_CODEC,
9487        );
9488        let child = child_context
9489            .start_child_workflow_avro_value(
9490                "collision.child",
9491                ChildWorkflowOptions::new("collision-workers"),
9492                AvroValue::Array(Vec::new()),
9493            )
9494            .await
9495            .expect("typed child collision result");
9496        assert_eq!(child.result, expected);
9497    }
9498
9499    #[tokio::test]
9500    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9501        let client = Client::new("http://127.0.0.1:8080").expect("client");
9502        let mut worker = Worker::new(client, "rust-workers");
9503        worker.register_replayed_workflow_avro_value(
9504            "typed.replayed",
9505            || (),
9506            |_ctx, input, _state| async move { Ok(input) },
9507        );
9508        worker.register_replayed_query_avro_value::<(), _, _>(
9509            "typed.replayed",
9510            "inspect",
9511            |ctx, _state, args| async move {
9512                let mut signals = ctx.signals_avro_value("collision");
9513                let signal = signals
9514                    .pop()
9515                    .map(AvroValue::Array)
9516                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9517                Ok(AvroValue::Array(vec![
9518                    ctx.workflow_input_avro_value().clone(),
9519                    signal,
9520                    args,
9521                ]))
9522            },
9523        );
9524        let arguments = AvroValue::Array(projection_collision_probe());
9525        let signal_arguments =
9526            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9527        let task = QueryTask {
9528            query_task_id: "query-typed-replay".to_string(),
9529            query_task_attempt: 1,
9530            lease_owner: Some("rust-worker".to_string()),
9531            workflow_id: Some("typed-replay".to_string()),
9532            run_id: Some("run-typed-replay".to_string()),
9533            workflow_type: "typed.replayed".to_string(),
9534            query_name: "inspect".to_string(),
9535            payload_codec: DEFAULT_CODEC.to_string(),
9536            workflow_arguments: Some(
9537                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9538            ),
9539            query_arguments: Some(
9540                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9541            ),
9542            history_events: vec![history_event(
9543                "SignalReceived",
9544                json!({
9545                    "signal_id": "collision-signal",
9546                    "signal_name": "collision",
9547                    "workflow_sequence": 1,
9548                    "payload_codec": DEFAULT_CODEC,
9549                    "arguments": signal_arguments,
9550                }),
9551            )],
9552            history_export: None,
9553            run_status: Some("completed".to_string()),
9554        };
9555
9556        assert_eq!(
9557            worker
9558                .execute_query_task(task)
9559                .await
9560                .expect("typed replay query"),
9561            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9562        );
9563    }
9564
9565    #[test]
9566    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9567        let value = BTreeMap::from([(1_i32, "integer key")]);
9568        let error = PayloadEnvelope::avro(&value)
9569            .expect_err("integer map keys must fail")
9570            .to_string();
9571
9572        assert!(error.contains("invalid_map_key"));
9573    }
9574
9575    #[test]
9576    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
9577        let envelope = PayloadEnvelope {
9578            codec: "json".to_string(),
9579            blob: r#"{"greeting":"hello"}"#.to_string(),
9580        };
9581
9582        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
9583        let diagnostic = error.to_string();
9584        assert!(diagnostic.contains("unsupported_payload_codec"));
9585        assert!(diagnostic.contains("codec=\"avro\""));
9586        assert!(diagnostic.contains("HTTP document transport"));
9587    }
9588
9589    #[test]
9590    fn untagged_json_payload_value_fails_closed() {
9591        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
9592            .expect_err("untagged JSON payload values must fail");
9593        let diagnostic = error.to_string();
9594        assert!(diagnostic.contains("unsupported_payload_codec"));
9595        assert!(diagnostic.contains("untagged durable payload"));
9596        assert!(diagnostic.contains("HTTP document transport"));
9597    }
9598
9599    #[test]
9600    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9601        let envelope = PayloadEnvelope {
9602            codec: DEFAULT_CODEC.to_string(),
9603            blob: BASE64.encode([0x01]),
9604        };
9605
9606        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9607        assert!(error.to_string().contains("invalid_payload_framing"));
9608    }
9609
9610    #[tokio::test]
9611    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
9612        let server = MockWorkerServer::start();
9613        let client = Client::builder(server.base_url())
9614            .timeout(Duration::from_secs(2))
9615            .build()
9616            .expect("client");
9617        let invalid_commands = [
9618            json!({
9619                "type": "complete_workflow",
9620                "result": {"codec": "json", "blob": null}
9621            }),
9622            json!({
9623                "type": "schedule_activity",
9624                "arguments": {"codec": "yaml", "blob": "ignored"}
9625            }),
9626            json!({
9627                "type": "start_child_workflow",
9628                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
9629            }),
9630            json!({"type": "continue_as_new", "arguments": []}),
9631            json!({"type": "complete_update"}),
9632            json!({"type": "record_side_effect", "result": null}),
9633            json!({
9634                "type": "start_service_operation",
9635                "payload_codec": DEFAULT_CODEC,
9636                "request_payload": "raw-avro-bytes"
9637            }),
9638        ];
9639
9640        for command in invalid_commands {
9641            let error = client
9642                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
9643                .await
9644                .expect_err("invalid durable payload must fail locally");
9645            let diagnostic = error.to_string();
9646            assert!(
9647                diagnostic.contains("unsupported_payload_codec")
9648                    || diagnostic.contains("invalid_payload_envelope")
9649                    || diagnostic.contains("untagged durable payload"),
9650                "unexpected validation diagnostic: {diagnostic}"
9651            );
9652        }
9653
9654        assert_eq!(
9655            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
9656            0,
9657            "invalid command payloads must not reach HTTP transport"
9658        );
9659    }
9660
9661    #[test]
9662    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
9663        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
9664        let commands = [
9665            json!({"type": "complete_workflow", "result": envelope.clone()}),
9666            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
9667            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
9668            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
9669            json!({"type": "complete_update", "result": envelope.clone()}),
9670            json!({"type": "record_side_effect", "result": envelope.clone()}),
9671            json!({
9672                "type": "start_service_operation",
9673                "payload_codec": DEFAULT_CODEC,
9674                "request_payload": envelope.clone()
9675            }),
9676            json!({
9677                "type": "complete_workflow",
9678                "result": envelope,
9679                "metadata": {
9680                    "codec": "json",
9681                    "payload_codec": "customer-codec",
9682                    "result": {"codec": "yaml", "blob": null}
9683                }
9684            }),
9685        ];
9686
9687        validate_workflow_task_commands(&commands)
9688            .expect("customer metadata must not become a protocol codec declaration");
9689    }
9690
9691    #[test]
9692    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
9693        assert_eq!(
9694            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
9695            AvroValue::Array(Vec::new())
9696        );
9697        assert_eq!(
9698            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
9699            AvroValue::Array(Vec::new())
9700        );
9701
9702        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
9703        signal.signal_name = Some("empty-signal".to_string());
9704        signal.signal_arguments = None;
9705        let decoded = decode_resume_signal(&signal)
9706            .expect("valid Avro signal")
9707            .expect("named signal resumes the workflow");
9708        assert!(decoded.arguments.is_empty());
9709    }
9710
9711    #[tokio::test]
9712    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
9713        let client = Client::new("http://127.0.0.1:8080").expect("client");
9714        let mut worker = Worker::new(client, "rust-workers");
9715        let handler_calls = Arc::new(AtomicUsize::new(0));
9716
9717        let calls = Arc::clone(&handler_calls);
9718        worker.register_workflow("codec.workflow", move |_ctx, _args| {
9719            calls.fetch_add(1, Ordering::SeqCst);
9720            async move { Ok(Value::Null) }
9721        });
9722        let calls = Arc::clone(&handler_calls);
9723        worker.register_activity("codec.activity", move |_ctx, _args| {
9724            calls.fetch_add(1, Ordering::SeqCst);
9725            async move { Ok(Value::Null) }
9726        });
9727        let calls = Arc::clone(&handler_calls);
9728        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
9729            calls.fetch_add(1, Ordering::SeqCst);
9730            async move { Ok(Value::Null) }
9731        });
9732        let calls = Arc::clone(&handler_calls);
9733        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9734            calls.fetch_add(1, Ordering::SeqCst);
9735            async move { Ok(Value::Null) }
9736        });
9737
9738        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9739        workflow.payload_codec = "json".to_string();
9740        workflow.arguments = None;
9741        let error = worker
9742            .execute_workflow_task(workflow)
9743            .expect_err("task codec must be checked before workflow invocation");
9744        assert!(error.to_string().contains("unsupported_payload_codec"));
9745
9746        let activity = ActivityTask {
9747            task_id: "activity-invalid-codec".to_string(),
9748            activity_attempt_id: None,
9749            attempt_id: None,
9750            activity_type: "codec.activity".to_string(),
9751            payload_codec: "unknown".to_string(),
9752            arguments: None,
9753            attempt_number: 1,
9754            lease_owner: None,
9755        };
9756        let error = worker
9757            .execute_activity_task(activity)
9758            .await
9759            .expect_err("task codec must be checked before activity invocation");
9760        assert!(error.to_string().contains("unsupported_payload_codec"));
9761
9762        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9763        update.workflow_update_id = Some("update-invalid-codec".to_string());
9764        update.update_name = Some("known".to_string());
9765        update.history_events.push(history_event(
9766            "UpdateAccepted",
9767            json!({
9768                "update_id": "update-invalid-codec",
9769                "update_name": "known",
9770                "arguments": {"codec": "json", "blob": null}
9771            }),
9772        ));
9773        let error = worker
9774            .execute_workflow_task(update)
9775            .expect_err("nested update codec must be checked before handler lookup");
9776        assert!(error.to_string().contains("unsupported_payload_codec"));
9777
9778        let query: QueryTask = serde_json::from_value(json!({
9779            "query_task_id": "query-invalid-codec",
9780            "workflow_type": "codec.workflow",
9781            "query_name": "known",
9782            "payload_codec": DEFAULT_CODEC,
9783            "workflow_arguments": null,
9784            "query_arguments": null,
9785            "history_export": {
9786                "payloads": {"codec": DEFAULT_CODEC},
9787                "signals": [{
9788                    "name": "empty",
9789                    "payload_codec": "json",
9790                    "arguments": null
9791                }]
9792            }
9793        }))
9794        .expect("query task");
9795        let failure = worker
9796            .execute_query_task(query)
9797            .await
9798            .expect_err("exported signal codec must be checked before query invocation");
9799        assert_eq!(failure.reason, "query_payload_decode_failed");
9800        assert!(failure.message.contains("unsupported_payload_codec"));
9801
9802        let exported_history: QueryTask = serde_json::from_value(json!({
9803            "query_task_id": "query-invalid-history-codec",
9804            "workflow_type": "codec.workflow",
9805            "query_name": "known",
9806            "payload_codec": DEFAULT_CODEC,
9807            "history_export": {
9808                "payloads": {"codec": DEFAULT_CODEC},
9809                "history_events": [{
9810                    "type": "ActivityCompleted",
9811                    "payload": {"payload_codec": "unknown", "result": null}
9812                }]
9813            }
9814        }))
9815        .expect("query task");
9816        let failure = worker
9817            .execute_query_task(exported_history)
9818            .await
9819            .expect_err("exported history codec must be checked before query invocation");
9820        assert_eq!(failure.reason, "query_payload_decode_failed");
9821        assert!(failure.message.contains("unsupported_payload_codec"));
9822        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
9823
9824        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
9825        unknown_workflow.arguments = None;
9826        unknown_workflow.history_events.push(history_event(
9827            "SignalReceived",
9828            json!({
9829                "signal_name": "empty",
9830                "payload_codec": "json",
9831                "arguments": null
9832            }),
9833        ));
9834        let error = worker
9835            .execute_workflow_task(unknown_workflow)
9836            .expect_err("history codec must precede unknown workflow outcome");
9837        assert!(error.to_string().contains("unsupported_payload_codec"));
9838
9839        let unknown_activity = ActivityTask {
9840            task_id: "activity-unknown".to_string(),
9841            activity_attempt_id: None,
9842            attempt_id: None,
9843            activity_type: "missing".to_string(),
9844            payload_codec: "json".to_string(),
9845            arguments: None,
9846            attempt_number: 1,
9847            lease_owner: None,
9848        };
9849        let error = worker
9850            .execute_activity_task(unknown_activity)
9851            .await
9852            .expect_err("codec must precede unknown activity outcome");
9853        assert!(error.to_string().contains("unsupported_payload_codec"));
9854
9855        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9856        unknown_update.payload_codec = "json".to_string();
9857        unknown_update.arguments = None;
9858        unknown_update.workflow_update_id = Some("update-unknown".to_string());
9859        unknown_update.update_name = Some("missing".to_string());
9860        let error = worker
9861            .execute_workflow_task(unknown_update)
9862            .expect_err("codec must precede fail_update shortcut");
9863        assert!(error.to_string().contains("unsupported_payload_codec"));
9864
9865        let unknown_query: QueryTask = serde_json::from_value(json!({
9866            "query_task_id": "query-unknown",
9867            "workflow_type": "missing",
9868            "query_name": "missing",
9869            "payload_codec": "json",
9870            "workflow_arguments": null,
9871            "query_arguments": null
9872        }))
9873        .expect("query task");
9874        let failure = worker
9875            .execute_query_task(unknown_query)
9876            .await
9877            .expect_err("codec must precede unknown query outcome");
9878        assert_eq!(failure.reason, "query_payload_decode_failed");
9879        assert!(failure.message.contains("unsupported_payload_codec"));
9880    }
9881
9882    #[tokio::test]
9883    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
9884        let client = Client::new("http://127.0.0.1:8080").expect("client");
9885        let worker = Worker::new(client, "rust-workers");
9886
9887        for event_type in ["SignalReceived", "SignalApplied"] {
9888            for (payload_field, codec) in [
9889                ("value", "json"),
9890                ("input", "unknown"),
9891                ("arguments", "json"),
9892            ] {
9893                let payload = json!({
9894                    "signal_name": "empty",
9895                    payload_field: {"codec": codec, "blob": null}
9896                });
9897                let workflow = workflow_task(
9898                    "missing",
9899                    vec![history_event(event_type, payload.clone())],
9900                    DEFAULT_CODEC,
9901                );
9902                let error = worker
9903                    .execute_workflow_task(workflow)
9904                    .expect_err("signal payload codec must precede unknown workflow outcome");
9905                assert!(
9906                    error.to_string().contains("unsupported_payload_codec"),
9907                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
9908                );
9909
9910                let query: QueryTask = serde_json::from_value(json!({
9911                    "query_task_id": format!("query-{event_type}-{payload_field}"),
9912                    "workflow_type": "missing",
9913                    "query_name": "missing",
9914                    "payload_codec": DEFAULT_CODEC,
9915                    "workflow_arguments": null,
9916                    "query_arguments": null,
9917                    "history_events": [{
9918                        "event_type": event_type,
9919                        "payload": payload
9920                    }]
9921                }))
9922                .expect("query task");
9923                let failure = worker
9924                    .execute_query_task(query)
9925                    .await
9926                    .expect_err("signal payload codec must precede unknown query outcome");
9927                assert_eq!(
9928                    failure.reason, "query_payload_decode_failed",
9929                    "{event_type}.{payload_field} returned an unrelated query outcome"
9930                );
9931                assert!(
9932                    failure.message.contains("unsupported_payload_codec"),
9933                    "{event_type}.{payload_field} returned an unrelated query error: {}",
9934                    failure.message
9935                );
9936            }
9937        }
9938    }
9939
9940    #[test]
9941    fn workflow_context_schedules_activity_until_completion_is_in_history() {
9942        let ctx = WorkflowContext {
9943            state: Arc::new(Mutex::new(
9944                WorkflowState::new_with_identity(
9945                    Vec::new(),
9946                    Some("wf-parent".to_string()),
9947                    Some("run-parent".to_string()),
9948                    "rust-workers".to_string(),
9949                    DEFAULT_CODEC.to_string(),
9950                    None,
9951                )
9952                .expect("workflow state"),
9953            )),
9954        };
9955
9956        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9957        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9958        assert!(matches!(
9959            call.as_mut().poll(&mut task_context),
9960            Poll::Pending
9961        ));
9962
9963        let commands = ctx.take_commands().expect("commands");
9964        assert_eq!(commands[0]["type"], "schedule_activity");
9965        assert_eq!(commands[0]["activity_type"], "hello.activity");
9966    }
9967
9968    #[test]
9969    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9970        let ctx = workflow_context(Vec::new());
9971        let options = ActivityOptions::new()
9972            .task_queue("payments")
9973            .retry_policy(
9974                ActivityRetryPolicy::new(4)
9975                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9976                    .non_retryable_error_type("ValidationError"),
9977            )
9978            .start_to_close_timeout(Duration::from_secs(120))
9979            .schedule_to_start_timeout(Duration::from_secs(10))
9980            .schedule_to_close_timeout(Duration::from_secs(300))
9981            .heartbeat_timeout(Duration::from_secs(15));
9982        let mut call = Box::pin(ctx.activity_with_options(
9983            "charge-card",
9984            options,
9985            json!([{"order_id": "o-1"}]),
9986        ));
9987        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9988
9989        assert!(matches!(
9990            call.as_mut().poll(&mut task_context),
9991            Poll::Pending
9992        ));
9993        assert!(matches!(
9994            call.as_mut().poll(&mut task_context),
9995            Poll::Pending
9996        ));
9997
9998        let commands = ctx.take_commands().expect("activity command");
9999        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
10000        assert_eq!(commands[0]["queue"], "payments");
10001        assert_eq!(
10002            commands[0]["retry_policy"],
10003            json!({
10004                "max_attempts": 4,
10005                "backoff_seconds": [1, 3, 9],
10006                "non_retryable_error_types": ["ValidationError"],
10007            })
10008        );
10009        assert_eq!(commands[0]["start_to_close_timeout"], 120);
10010        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
10011        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
10012        assert_eq!(commands[0]["heartbeat_timeout"], 15);
10013    }
10014
10015    #[test]
10016    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
10017        let ctx = workflow_context(Vec::new());
10018        let options = ActivityOptions::new().retry_policy(
10019            ActivityRetryPolicy::new(3)
10020                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
10021        );
10022        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10023        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10024
10025        assert!(matches!(
10026            call.as_mut().poll(&mut task_context),
10027            Poll::Pending
10028        ));
10029        assert_eq!(
10030            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
10031            json!([1, 2])
10032        );
10033    }
10034
10035    #[test]
10036    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
10037        let cases = [
10038            (
10039                ActivityOptions::new().task_queue("  "),
10040                ActivityOptionsErrorKind::EmptyTaskQueue,
10041            ),
10042            (
10043                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
10044                ActivityOptionsErrorKind::EmptyRetryPolicy,
10045            ),
10046            (
10047                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
10048                ActivityOptionsErrorKind::InvalidMaxAttempts,
10049            ),
10050            (
10051                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
10052                    max_attempts: None,
10053                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
10054                    non_retryable_error_types: Vec::new(),
10055                }),
10056                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
10057            ),
10058            (
10059                ActivityOptions::new().retry_policy(
10060                    ActivityRetryPolicy::new(2)
10061                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
10062                ),
10063                ActivityOptionsErrorKind::TooManyBackoffIntervals,
10064            ),
10065            (
10066                ActivityOptions::new().retry_policy(
10067                    ActivityRetryPolicy::new(2).exponential_backoff(
10068                        Duration::from_secs(1),
10069                        0,
10070                        None,
10071                    ),
10072                ),
10073                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
10074            ),
10075            (
10076                ActivityOptions::new()
10077                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
10078                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
10079            ),
10080            (
10081                ActivityOptions::new().retry_policy(
10082                    ActivityRetryPolicy::new(10_002).exponential_backoff(
10083                        Duration::from_secs(1),
10084                        1,
10085                        None,
10086                    ),
10087                ),
10088                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
10089            ),
10090            (
10091                ActivityOptions::new().retry_policy(
10092                    ActivityRetryPolicy::new(2)
10093                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
10094                ),
10095                ActivityOptionsErrorKind::BackoffOverflow,
10096            ),
10097        ];
10098
10099        for (options, expected_kind) in cases {
10100            let ctx = workflow_context(Vec::new());
10101            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10102            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10103            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
10104                call.as_mut().poll(&mut task_context)
10105            else {
10106                panic!("expected typed activity validation error");
10107            };
10108            assert_eq!(error.kind, expected_kind);
10109            assert!(ctx.take_commands().expect("commands").is_empty());
10110        }
10111    }
10112
10113    #[test]
10114    fn activity_options_validate_positive_and_ordered_timeouts() {
10115        let zero_timeout_cases = [
10116            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
10117            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
10118            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
10119            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
10120        ];
10121        for options in zero_timeout_cases {
10122            assert_eq!(
10123                options.validate().expect_err("zero timeout").kind,
10124                ActivityOptionsErrorKind::TimeoutNotPositive
10125            );
10126        }
10127
10128        let ordering_cases = [
10129            ActivityOptions::new()
10130                .heartbeat_timeout(Duration::from_secs(11))
10131                .start_to_close_timeout(Duration::from_secs(10)),
10132            ActivityOptions::new()
10133                .start_to_close_timeout(Duration::from_secs(31))
10134                .schedule_to_close_timeout(Duration::from_secs(30)),
10135            ActivityOptions::new()
10136                .schedule_to_start_timeout(Duration::from_secs(31))
10137                .schedule_to_close_timeout(Duration::from_secs(30)),
10138        ];
10139        for options in ordering_cases {
10140            assert_eq!(
10141                options.validate().expect_err("timeout order").kind,
10142                ActivityOptionsErrorKind::TimeoutOrder
10143            );
10144        }
10145
10146        assert_eq!(
10147            ActivityOptions::new()
10148                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
10149                .validate()
10150                .expect_err("protocol integer overflow")
10151                .kind,
10152            ActivityOptionsErrorKind::TimeoutOverflow
10153        );
10154    }
10155
10156    #[test]
10157    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
10158        let ctx = workflow_context(completed_retry_activity_history());
10159        let mut call =
10160            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10161        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10162
10163        assert!(matches!(
10164            call.as_mut().poll(&mut task_context),
10165            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10166        ));
10167        assert!(ctx.take_commands().expect("commands").is_empty());
10168        ctx.ensure_history_consumed().expect("history consumed");
10169    }
10170
10171    #[test]
10172    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
10173        let mut options = retry_activity_options();
10174        options
10175            .retry_policy
10176            .as_mut()
10177            .expect("retry policy")
10178            .non_retryable_error_types
10179            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
10180
10181        let new_ctx = workflow_context(Vec::new());
10182        let mut new_call =
10183            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
10184        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10185        assert!(matches!(
10186            new_call.as_mut().poll(&mut task_context),
10187            Poll::Pending
10188        ));
10189        let commands = new_ctx.take_commands().expect("commands");
10190        assert_eq!(commands.len(), 1);
10191        assert_eq!(
10192            commands[0]["retry_policy"]["non_retryable_error_types"],
10193            json!(["PermanentError"])
10194        );
10195
10196        let replay_ctx = workflow_context(completed_retry_activity_history());
10197        let mut replay_call =
10198            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
10199        assert!(matches!(
10200            replay_call.as_mut().poll(&mut task_context),
10201            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10202        ));
10203        assert!(replay_ctx.take_commands().expect("commands").is_empty());
10204        replay_ctx
10205            .ensure_history_consumed()
10206            .expect("history consumed");
10207    }
10208
10209    #[test]
10210    fn replayed_intermediate_retry_remains_pending_across_restarts() {
10211        let history = completed_retry_activity_history()
10212            .into_iter()
10213            .take(3)
10214            .collect::<Vec<_>>();
10215
10216        for _restart in 0..2 {
10217            let ctx = workflow_context(history.clone());
10218            let mut call =
10219                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10220            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10221            assert!(matches!(
10222                call.as_mut().poll(&mut task_context),
10223                Poll::Pending
10224            ));
10225            assert!(ctx.take_commands().expect("commands").is_empty());
10226        }
10227    }
10228
10229    #[test]
10230    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
10231        let mut changed_queue = retry_activity_options();
10232        changed_queue.task_queue = Some("different-queue".to_string());
10233
10234        let mut changed_max_attempts = retry_activity_options();
10235        let retry_policy = changed_max_attempts
10236            .retry_policy
10237            .as_mut()
10238            .expect("retry policy");
10239        retry_policy.max_attempts = Some(4);
10240
10241        let mut changed_backoff = retry_activity_options();
10242        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
10243        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
10244            Duration::from_secs(3),
10245            Duration::from_secs(4),
10246        ]));
10247
10248        let mut changed_non_retryable_types = retry_activity_options();
10249        let retry_policy = changed_non_retryable_types
10250            .retry_policy
10251            .as_mut()
10252            .expect("retry policy");
10253        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
10254
10255        let mut changed_start_to_close = retry_activity_options();
10256        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
10257        let mut changed_schedule_to_start = retry_activity_options();
10258        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
10259        let mut changed_schedule_to_close = retry_activity_options();
10260        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
10261        let mut changed_heartbeat = retry_activity_options();
10262        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
10263
10264        let cases = [
10265            (changed_queue, "activity_task_queue_mismatch"),
10266            (changed_max_attempts, "activity_retry_policy_mismatch"),
10267            (changed_backoff, "activity_retry_policy_mismatch"),
10268            (
10269                changed_non_retryable_types,
10270                "activity_retry_policy_mismatch",
10271            ),
10272            (changed_start_to_close, "activity_retry_policy_mismatch"),
10273            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
10274            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
10275            (changed_heartbeat, "activity_retry_policy_mismatch"),
10276        ];
10277
10278        for (options, expected_reason) in cases {
10279            let ctx = workflow_context(completed_retry_activity_history());
10280            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
10281            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10282            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10283                call.as_mut().poll(&mut task_context)
10284            else {
10285                panic!("changed activity options must fail replay");
10286            };
10287            assert_eq!(failure.reason, expected_reason);
10288            assert_eq!(failure.sequence, Some(1));
10289            assert!(ctx.take_commands().expect("commands").is_empty());
10290        }
10291    }
10292
10293    #[test]
10294    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
10295        let cases = [
10296            (
10297                "execution_mode",
10298                json!("local"),
10299                "activity_execution_mode_mismatch",
10300            ),
10301            (
10302                "snapshot_version",
10303                json!(2),
10304                "activity_retry_policy_mismatch",
10305            ),
10306        ];
10307
10308        for (field, value, expected_reason) in cases {
10309            let mut history = completed_retry_activity_history();
10310            let activity = history[0].payload["activity"]
10311                .as_object_mut()
10312                .expect("activity snapshot");
10313            if field == "execution_mode" {
10314                activity.insert(field.to_string(), value);
10315            } else {
10316                activity["retry_policy"]
10317                    .as_object_mut()
10318                    .expect("retry snapshot")
10319                    .insert(field.to_string(), value);
10320            }
10321
10322            let ctx = workflow_context(history);
10323            let mut call =
10324                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10325            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10326            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10327                call.as_mut().poll(&mut task_context)
10328            else {
10329                panic!("changed {field} must fail replay");
10330            };
10331            assert_eq!(failure.reason, expected_reason);
10332            assert_eq!(failure.sequence, Some(1));
10333            assert!(ctx.take_commands().expect("commands").is_empty());
10334        }
10335    }
10336
10337    #[test]
10338    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
10339        let mut history = completed_retry_activity_history();
10340        let activity = history[0].payload["activity"]
10341            .as_object_mut()
10342            .expect("activity snapshot");
10343        activity.remove("execution_mode");
10344        activity.remove("retry_policy");
10345
10346        let mut current = retry_activity_options();
10347        current.start_to_close_timeout = Some(Duration::from_secs(45));
10348        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
10349        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
10350        current.heartbeat_timeout = Some(Duration::from_secs(12));
10351
10352        let ctx = workflow_context(history);
10353        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
10354        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10355        assert!(matches!(
10356            call.as_mut().poll(&mut task_context),
10357            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10358        ));
10359        assert!(ctx.take_commands().expect("commands").is_empty());
10360        ctx.ensure_history_consumed().expect("history consumed");
10361    }
10362
10363    #[test]
10364    fn terminal_activity_failed_after_start_returns_typed_failure() {
10365        let history = vec![
10366            history_event(
10367                "ActivityScheduled",
10368                json!({
10369                    "sequence": 1,
10370                    "activity_type": "flaky",
10371                    "activity_execution_id": "act-terminal",
10372                    "activity": {
10373                        "id": "act-terminal",
10374                        "sequence": 1,
10375                        "type": "flaky",
10376                        "queue": "critical-activities",
10377                        "retry_policy": {
10378                            "snapshot_version": 1,
10379                            "max_attempts": 3,
10380                            "backoff_seconds": [2, 4],
10381                            "non_retryable_error_types": ["PermanentError"]
10382                        }
10383                    }
10384                }),
10385            ),
10386            history_event(
10387                "ActivityStarted",
10388                json!({
10389                    "sequence": 1,
10390                    "activity_type": "flaky",
10391                    "activity_execution_id": "act-terminal",
10392                    "activity_attempt_id": "attempt-1",
10393                    "attempt_number": 1
10394                }),
10395            ),
10396            history_event(
10397                "ActivityFailed",
10398                json!({
10399                    "sequence": 1,
10400                    "activity_type": "flaky",
10401                    "activity_execution_id": "act-terminal",
10402                    "activity_attempt_id": "attempt-1",
10403                    "attempt_number": 1,
10404                    "failure_id": "failure-terminal",
10405                    "failure_category": "activity",
10406                    "exception_type": "PermanentError",
10407                    "message": "cannot retry",
10408                    "non_retryable": true
10409                }),
10410            ),
10411        ];
10412        let ctx = workflow_context(history);
10413        let mut call =
10414            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10415        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10416
10417        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10418            call.as_mut().poll(&mut task_context)
10419        else {
10420            panic!("terminal ActivityFailed must settle the activity future");
10421        };
10422        assert_eq!(failure.kind, ActivityFailureKind::Failed);
10423        assert_eq!(
10424            failure.activity_execution_id.as_deref(),
10425            Some("act-terminal")
10426        );
10427        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
10428        assert!(failure.non_retryable);
10429        assert!(ctx.take_commands().expect("commands").is_empty());
10430        ctx.ensure_history_consumed().expect("history consumed");
10431    }
10432
10433    #[test]
10434    fn activity_terminal_events_return_machine_readable_failures() {
10435        let cases = [
10436            (
10437                "ActivityFailed",
10438                json!({
10439                    "sequence": 1,
10440                    "activity_type": "charge-card",
10441                    "activity_execution_id": "act-1",
10442                    "activity_attempt_id": "attempt-2",
10443                    "attempt_number": 2,
10444                    "failure_id": "failure-1",
10445                    "failure_category": "activity",
10446                    "exception_type": "PaymentDeclined",
10447                    "exception_class": "payments.PaymentDeclined",
10448                    "message": "card declined",
10449                    "non_retryable": true
10450                }),
10451                ActivityFailureKind::Failed,
10452                "activity",
10453            ),
10454            (
10455                "ActivityCancelled",
10456                json!({
10457                    "sequence": 1,
10458                    "activity_type": "charge-card",
10459                    "activity_execution_id": "act-1",
10460                    "activity_attempt_id": "attempt-1"
10461                }),
10462                ActivityFailureKind::Cancelled,
10463                "cancelled",
10464            ),
10465        ];
10466
10467        for (event_type, payload, expected_kind, expected_reason) in cases {
10468            let ctx = workflow_context(vec![history_event(event_type, payload)]);
10469            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
10470            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10471            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10472                call.as_mut().poll(&mut task_context)
10473            else {
10474                panic!("expected terminal activity failure");
10475            };
10476            assert_eq!(failure.kind, expected_kind);
10477            assert_eq!(failure.reason, expected_reason);
10478            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
10479            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
10480        }
10481    }
10482
10483    #[test]
10484    fn every_activity_timeout_class_is_typed() {
10485        for timeout_kind in [
10486            "start_to_close",
10487            "schedule_to_start",
10488            "schedule_to_close",
10489            "heartbeat",
10490        ] {
10491            let ctx = workflow_context(vec![history_event(
10492                "ActivityTimedOut",
10493                json!({
10494                    "sequence": 1,
10495                    "activity_type": "slow",
10496                    "activity_execution_id": "act-timeout",
10497                    "activity_attempt_id": "attempt-timeout",
10498                    "failure_category": "timeout",
10499                    "timeout_kind": timeout_kind,
10500                    "message": "deadline expired"
10501                }),
10502            )]);
10503            let mut call = Box::pin(ctx.activity("slow", json!([])));
10504            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10505            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10506                call.as_mut().poll(&mut task_context)
10507            else {
10508                panic!("expected timeout failure");
10509            };
10510            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
10511            assert_eq!(failure.reason, timeout_kind);
10512            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
10513            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
10514        }
10515    }
10516
10517    #[test]
10518    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
10519        let ctx = workflow_context(Vec::new());
10520        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
10521        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10522
10523        assert!(matches!(
10524            sleep.as_mut().poll(&mut task_context),
10525            Poll::Pending
10526        ));
10527        assert!(matches!(
10528            sleep.as_mut().poll(&mut task_context),
10529            Poll::Pending
10530        ));
10531
10532        let commands = ctx.take_commands().expect("timer command");
10533        assert_eq!(
10534            commands,
10535            vec![json!({
10536                "type": "start_timer",
10537                "delay_seconds": 2,
10538            })]
10539        );
10540    }
10541
10542    #[test]
10543    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
10544        let history = vec![
10545            history_event(
10546                "TimerScheduled",
10547                json!({
10548                    "sequence": 1,
10549                    "timer_id": "timer-1",
10550                    "delay_seconds": 5,
10551                    "fire_at": "2026-07-11T12:00:05Z",
10552                }),
10553            ),
10554            history_event(
10555                "TimerFired",
10556                json!({
10557                    "sequence": 1,
10558                    "timer_id": "timer-1",
10559                    "delay_seconds": 5,
10560                    "fire_at": "2026-07-11T12:00:05Z",
10561                    "fired_at": "2026-07-11T12:00:05Z",
10562                }),
10563            ),
10564        ];
10565
10566        for _restart in 0..2 {
10567            let ctx = workflow_context(history.clone());
10568            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
10569            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10570            assert!(matches!(
10571                sleep.as_mut().poll(&mut task_context),
10572                Poll::Ready(Ok(()))
10573            ));
10574            assert!(ctx.take_commands().expect("commands").is_empty());
10575            ctx.ensure_history_consumed().expect("history consumed");
10576        }
10577    }
10578
10579    #[test]
10580    fn workflow_sleep_rejects_changed_delay_during_replay() {
10581        let ctx = workflow_context(vec![
10582            history_event(
10583                "TimerScheduled",
10584                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10585            ),
10586            history_event(
10587                "TimerFired",
10588                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10589            ),
10590        ]);
10591        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10592        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10593
10594        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10595            sleep.as_mut().poll(&mut task_context)
10596        else {
10597            panic!("changed timer delay must be rejected");
10598        };
10599        assert_eq!(failure.reason, "timer_delay_mismatch");
10600        assert_eq!(failure.sequence, Some(1));
10601    }
10602
10603    #[test]
10604    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10605        let lone_fire = WorkflowState::new(
10606            vec![history_event(
10607                "TimerFired",
10608                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10609            )],
10610            "rust-workers".to_string(),
10611            DEFAULT_CODEC.to_string(),
10612            None,
10613        )
10614        .expect_err("TimerFired requires TimerScheduled");
10615        assert!(matches!(
10616            lone_fire,
10617            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10618                if reason == "timer_schedule_missing_or_duplicate"
10619        ));
10620
10621        let wrong_identity = WorkflowState::new(
10622            vec![
10623                history_event(
10624                    "TimerScheduled",
10625                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10626                ),
10627                history_event(
10628                    "TimerFired",
10629                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10630                ),
10631            ],
10632            "rust-workers".to_string(),
10633            DEFAULT_CODEC.to_string(),
10634            None,
10635        )
10636        .expect_err("fire must match scheduled timer identity");
10637        assert!(matches!(
10638            wrong_identity,
10639            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10640                if reason == "timer_identity_mismatch"
10641        ));
10642
10643        let duplicate_fire = WorkflowState::new(
10644            vec![
10645                history_event(
10646                    "TimerScheduled",
10647                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10648                ),
10649                history_event(
10650                    "TimerFired",
10651                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10652                ),
10653                history_event(
10654                    "TimerFired",
10655                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10656                ),
10657            ],
10658            "rust-workers".to_string(),
10659            DEFAULT_CODEC.to_string(),
10660            None,
10661        )
10662        .expect_err("a durable timer cannot fire twice");
10663        assert!(matches!(
10664            duplicate_fire,
10665            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10666                if reason == "duplicate_timer_fire"
10667        ));
10668
10669        let wrong_fired_delay = WorkflowState::new(
10670            vec![
10671                history_event(
10672                    "TimerScheduled",
10673                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10674                ),
10675                history_event(
10676                    "TimerFired",
10677                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10678                ),
10679            ],
10680            "rust-workers".to_string(),
10681            DEFAULT_CODEC.to_string(),
10682            None,
10683        )
10684        .expect_err("timer schedule and fire delays must agree");
10685        assert!(matches!(
10686            wrong_fired_delay,
10687            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10688                if reason == "timer_history_delay_mismatch"
10689        ));
10690    }
10691
10692    #[test]
10693    fn replay_rejects_activity_moved_before_recorded_timer() {
10694        let ctx = workflow_context(vec![
10695            history_event(
10696                "TimerScheduled",
10697                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10698            ),
10699            history_event(
10700                "TimerFired",
10701                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10702            ),
10703            history_event(
10704                "ActivityCompleted",
10705                json!({
10706                    "sequence": 2,
10707                    "activity_type": "after-timer",
10708                    "payload_codec": DEFAULT_CODEC,
10709                    "result": fixture_envelope(json!("done")),
10710                }),
10711            ),
10712        ]);
10713        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10714        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10715
10716        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10717            activity.as_mut().poll(&mut task_context)
10718        else {
10719            panic!("reordered durable command must be rejected");
10720        };
10721        assert_eq!(failure.reason, "recorded_command_mismatch");
10722        assert_eq!(failure.sequence, Some(1));
10723        assert_eq!(failure.expected.as_deref(), Some("timer"));
10724        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10725    }
10726
10727    #[test]
10728    fn workflow_context_emits_a_typed_named_signal_wait() {
10729        let ctx = workflow_context(Vec::new());
10730        let mut signal = Box::pin(ctx.wait_signal("finish"));
10731        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10732
10733        assert!(matches!(
10734            signal.as_mut().poll(&mut task_context),
10735            Poll::Pending
10736        ));
10737        assert_eq!(
10738            ctx.take_commands().expect("signal-wait command"),
10739            vec![json!({
10740                "type": "open_signal_wait",
10741                "signal_name": "finish",
10742            })]
10743        );
10744    }
10745
10746    #[test]
10747    fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10748        let ctx = workflow_context(vec![
10749            history_event(
10750                "ConditionWaitOpened",
10751                json!({"sequence": 1, "condition_key": "signal:finish"}),
10752            ),
10753            history_event(
10754                "ConditionWaitSatisfied",
10755                json!({"sequence": 1, "condition_key": "signal:finish"}),
10756            ),
10757            history_event(
10758                "SignalReceived",
10759                json!({"signal_name": "finish", "arguments": []}),
10760            ),
10761        ]);
10762        let mut signal = Box::pin(ctx.wait_signal("finish"));
10763        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10764
10765        assert!(matches!(
10766            signal.as_mut().poll(&mut task_context),
10767            Poll::Pending
10768        ));
10769        assert_eq!(
10770            ctx.take_commands().expect("typed signal-wait command"),
10771            vec![json!({
10772                "type": "open_signal_wait",
10773                "signal_name": "finish",
10774            })]
10775        );
10776    }
10777
10778    #[test]
10779    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10780        let signal_then_timer = vec![
10781            history_event(
10782                "SignalWaitOpened",
10783                json!({"sequence": 1, "signal_name": "go"}),
10784            ),
10785            history_event(
10786                "SignalApplied",
10787                json!({
10788                    "sequence": 1,
10789                    "signal_name": "go",
10790                    "value": fixture_envelope(json!(["now"])),
10791                }),
10792            ),
10793            history_event(
10794                "TimerScheduled",
10795                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10796            ),
10797            history_event(
10798                "TimerFired",
10799                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10800            ),
10801        ];
10802
10803        let ctx = workflow_context(signal_then_timer.clone());
10804        let mut signal = Box::pin(ctx.wait_signal("go"));
10805        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10806        assert!(matches!(
10807            signal.as_mut().poll(&mut task_context),
10808            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10809        ));
10810        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10811        assert!(matches!(
10812            timer.as_mut().poll(&mut task_context),
10813            Poll::Ready(Ok(()))
10814        ));
10815        ctx.ensure_history_consumed()
10816            .expect("signal and timer history consumed in order");
10817
10818        let reordered = workflow_context(signal_then_timer);
10819        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10820        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10821            timer_first.as_mut().poll(&mut task_context)
10822        else {
10823            panic!("timer cannot consume signal-wait-first history");
10824        };
10825        assert_eq!(failure.reason, "recorded_command_mismatch");
10826        assert_eq!(failure.sequence, Some(1));
10827        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10828
10829        let timer_then_signal = vec![
10830            history_event(
10831                "TimerScheduled",
10832                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10833            ),
10834            history_event(
10835                "TimerFired",
10836                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10837            ),
10838            history_event(
10839                "SignalWaitOpened",
10840                json!({"sequence": 2, "signal_name": "go"}),
10841            ),
10842            history_event(
10843                "SignalApplied",
10844                json!({
10845                    "sequence": 2,
10846                    "signal_name": "go",
10847                    "value": fixture_envelope(json!([])),
10848                }),
10849            ),
10850        ];
10851        let reordered = workflow_context(timer_then_signal);
10852        let mut signal_first = Box::pin(reordered.wait_signal("go"));
10853        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10854            signal_first.as_mut().poll(&mut task_context)
10855        else {
10856            panic!("signal wait cannot consume timer-first history");
10857        };
10858        assert_eq!(failure.reason, "recorded_command_mismatch");
10859        assert_eq!(failure.sequence, Some(1));
10860        assert_eq!(failure.expected.as_deref(), Some("timer"));
10861    }
10862
10863    #[test]
10864    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10865        let duplicate_timer = WorkflowState::new(
10866            vec![
10867                history_event(
10868                    "TimerScheduled",
10869                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10870                ),
10871                history_event(
10872                    "TimerScheduled",
10873                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10874                ),
10875            ],
10876            "rust-workers".to_string(),
10877            DEFAULT_CODEC.to_string(),
10878            None,
10879        )
10880        .expect_err("one workflow sequence cannot schedule two timers");
10881        assert!(matches!(
10882            duplicate_timer,
10883            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10884                if reason == "timer_schedule_missing_or_duplicate"
10885        ));
10886
10887        let colliding_kinds = WorkflowState::new(
10888            vec![
10889                history_event(
10890                    "TimerScheduled",
10891                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10892                ),
10893                history_event(
10894                    "ActivityCompleted",
10895                    json!({"sequence": 1, "activity_type": "same-sequence"}),
10896                ),
10897            ],
10898            "rust-workers".to_string(),
10899            DEFAULT_CODEC.to_string(),
10900            None,
10901        )
10902        .expect_err("one workflow sequence cannot identify two command kinds");
10903        assert!(matches!(
10904            colliding_kinds,
10905            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10906                if reason == "durable_command_sequence_collision"
10907        ));
10908
10909        let duplicate_signal_wait = WorkflowState::new(
10910            vec![
10911                history_event(
10912                    "SignalWaitOpened",
10913                    json!({"sequence": 1, "signal_name": "go"}),
10914                ),
10915                history_event(
10916                    "SignalWaitOpened",
10917                    json!({"sequence": 1, "signal_name": "go"}),
10918                ),
10919            ],
10920            "rust-workers".to_string(),
10921            DEFAULT_CODEC.to_string(),
10922            None,
10923        )
10924        .expect_err("one workflow sequence cannot open two signal waits");
10925        assert!(matches!(
10926            duplicate_signal_wait,
10927            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10928                if reason == "signal_wait_open_missing_or_duplicate"
10929        ));
10930    }
10931
10932    #[test]
10933    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10934        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
10935            .expect("side-effect result");
10936        let ctx = workflow_context(vec![history_event(
10937            "SideEffectRecorded",
10938            json!({"sequence": 99, "result": result}),
10939        )]);
10940
10941        let replayed: Value = ctx
10942            .side_effect(|| panic!("recorded side effect must not run"))
10943            .expect("positive global workflow sequence is valid");
10944        assert_eq!(replayed, json!({"captured": true}));
10945        ctx.ensure_history_consumed().expect("history consumed");
10946    }
10947
10948    #[test]
10949    fn workflow_history_rejects_zero_and_descending_command_sequences() {
10950        let result =
10951            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
10952        let zero = WorkflowState::new(
10953            vec![history_event(
10954                "SideEffectRecorded",
10955                json!({"sequence": 0, "result": result.clone()}),
10956            )],
10957            "rust-workers".to_string(),
10958            DEFAULT_CODEC.to_string(),
10959            None,
10960        )
10961        .expect_err("durable command sequences must be positive");
10962        assert!(matches!(
10963            zero,
10964            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10965                if reason == "durable_command_sequence_invalid"
10966        ));
10967
10968        let descending = WorkflowState::new(
10969            vec![
10970                history_event(
10971                    "SideEffectRecorded",
10972                    json!({"sequence": 3, "result": result}),
10973                ),
10974                history_event(
10975                    "VersionMarkerRecorded",
10976                    json!({
10977                        "sequence": 2,
10978                        "change_id": "descending-marker",
10979                        "version": 1,
10980                        "min_supported": 1,
10981                        "max_supported": 1,
10982                    }),
10983                ),
10984            ],
10985            "rust-workers".to_string(),
10986            DEFAULT_CODEC.to_string(),
10987            None,
10988        )
10989        .expect_err("new durable commands must remain strictly ordered");
10990        let Error::NonDeterministicReplay(failure) = descending else {
10991            panic!("expected typed replay failure");
10992        };
10993        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10994        assert_eq!(failure.sequence, Some(2));
10995        assert_eq!(
10996            failure.expected.as_deref(),
10997            Some("workflow sequence greater than 3")
10998        );
10999        assert_eq!(failure.actual.as_deref(), Some("2"));
11000    }
11001
11002    #[test]
11003    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
11004        fn worker() -> Worker {
11005            let client = Client::new("http://127.0.0.1:8080").expect("client");
11006            let mut worker = Worker::new(client, "rust-workers");
11007            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
11008                ctx.wait_signal("finish").await?;
11009                let marker: String =
11010                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
11011                assert_eq!(marker, "after-finish");
11012                Ok(json!("finished"))
11013            });
11014            worker
11015        }
11016
11017        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
11018            .expect("side-effect result");
11019        let task = workflow_task(
11020            "rust.finish-after-gaps",
11021            vec![
11022                history_event(
11023                    "SignalWaitOpened",
11024                    json!({"sequence": 1, "signal_name": "finish"}),
11025                ),
11026                history_event(
11027                    "SignalReceived",
11028                    json!({
11029                        "signal_id": "increment-3",
11030                        "signal_name": "increment",
11031                        "workflow_sequence": 2,
11032                        "payload_codec": DEFAULT_CODEC,
11033                        "arguments": fixture_envelope(json!([3])),
11034                    }),
11035                ),
11036                history_event(
11037                    "SignalReceived",
11038                    json!({
11039                        "signal_id": "increment-5",
11040                        "signal_name": "increment",
11041                        "workflow_sequence": 3,
11042                        "payload_codec": DEFAULT_CODEC,
11043                        "arguments": fixture_envelope(json!([5])),
11044                    }),
11045                ),
11046                history_event(
11047                    "SignalReceived",
11048                    json!({
11049                        "signal_id": "finish",
11050                        "signal_name": "finish",
11051                        "workflow_sequence": 4,
11052                        "payload_codec": DEFAULT_CODEC,
11053                        "arguments": fixture_envelope(json!([])),
11054                    }),
11055                ),
11056                history_event(
11057                    "SignalApplied",
11058                    json!({
11059                        "sequence": 1,
11060                        "signal_id": "finish",
11061                        "signal_name": "finish",
11062                        "payload_codec": DEFAULT_CODEC,
11063                        "value": fixture_envelope(json!([])),
11064                    }),
11065                ),
11066                history_event(
11067                    "SideEffectRecorded",
11068                    json!({"sequence": 5, "result": marker}),
11069                ),
11070            ],
11071            DEFAULT_CODEC,
11072        );
11073
11074        for _original_or_cold_worker in 0..2 {
11075            let commands = worker()
11076                .execute_workflow_task(task.clone())
11077                .expect("signal gaps preserve deterministic replay");
11078            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
11079            assert_eq!(commands[0]["type"], "complete_workflow");
11080            assert_eq!(
11081                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
11082                json!("finished")
11083            );
11084        }
11085    }
11086
11087    #[test]
11088    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
11089        let ctx = workflow_context(Vec::new());
11090        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
11091        let mut task_context = TaskContext::from_waker(noop_waker_ref());
11092        assert!(matches!(
11093            sleep.as_mut().poll(&mut task_context),
11094            Poll::Ready(Err(Error::TimerDurationOverflow))
11095        ));
11096        assert!(ctx.take_commands().expect("commands").is_empty());
11097    }
11098
11099    #[test]
11100    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
11101        let client = Client::new("http://127.0.0.1:8080").expect("client");
11102        let mut worker = Worker::new(client, "rust-workers");
11103        worker.register_workflow("rust.timer", |ctx, _input| async move {
11104            ctx.sleep(Duration::from_secs(5)).await?;
11105            ctx.activity("after-timer", json!([])).await
11106        });
11107
11108        let task = |history_events| WorkflowTask {
11109            task_id: "wft-rust-timer-1".to_string(),
11110            workflow_id: Some("wf-rust-timer".to_string()),
11111            run_id: Some("run-rust-timer".to_string()),
11112            workflow_type: "rust.timer".to_string(),
11113            payload_codec: DEFAULT_CODEC.to_string(),
11114            arguments: Some(
11115                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11116            ),
11117            history_events,
11118            total_history_events: None,
11119            history_size_bytes: None,
11120            continue_as_new_recommended: None,
11121            history_budget_pressure: None,
11122            next_history_page_token: None,
11123            workflow_task_attempt: 1,
11124            workflow_signal_id: None,
11125            signal_name: None,
11126            signal_arguments: None,
11127            workflow_update_id: None,
11128            update_name: None,
11129            lease_owner: Some("rust-worker".to_string()),
11130        };
11131
11132        let initial = worker
11133            .execute_workflow_task(task(Vec::new()))
11134            .expect("initial timer task");
11135        assert_eq!(
11136            initial,
11137            vec![json!({"type": "start_timer", "delay_seconds": 5})]
11138        );
11139
11140        let activity_result =
11141            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
11142        let replayed = worker
11143            .execute_workflow_task(task(vec![
11144                history_event(
11145                    "TimerScheduled",
11146                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11147                ),
11148                history_event(
11149                    "TimerFired",
11150                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11151                ),
11152                history_event(
11153                    "ActivityCompleted",
11154                    json!({
11155                        "sequence": 2,
11156                        "activity_type": "after-timer",
11157                        "payload_codec": DEFAULT_CODEC,
11158                        "result": activity_result,
11159                    }),
11160                ),
11161            ]))
11162            .expect("replayed workflow task");
11163        assert_eq!(replayed.len(), 1);
11164        assert_eq!(replayed[0]["type"], "complete_workflow");
11165        assert_eq!(
11166            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
11167            json!("done")
11168        );
11169    }
11170
11171    #[test]
11172    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
11173        let client = Client::new("http://127.0.0.1:8080").expect("client");
11174        let mut worker = Worker::new(client, "rust-workers");
11175        worker.register_workflow("rust.continue", |ctx, _input| async move {
11176            ctx.continue_as_new_with_options(
11177                ContinueAsNewOptions::new()
11178                    .workflow_type("rust.next")
11179                    .task_queue("next-workers"),
11180                json!([2, {"cursor": "next"}]),
11181            )
11182        });
11183
11184        let commands = worker
11185            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
11186            .expect("continue-as-new command");
11187
11188        assert_eq!(commands.len(), 1);
11189        assert_eq!(commands[0]["type"], "continue_as_new");
11190        assert_eq!(commands[0]["workflow_type"], "rust.next");
11191        assert_eq!(commands[0]["queue"], "next-workers");
11192        assert_eq!(
11193            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
11194                .expect("continue-as-new arguments"),
11195            json!([2, {"cursor": "next"}])
11196        );
11197    }
11198
11199    #[test]
11200    fn continue_as_new_preserves_typed_arguments() {
11201        let client = Client::new("http://127.0.0.1:8080").expect("client");
11202        let mut worker = Worker::new(client, "rust-workers");
11203        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
11204            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
11205            unreachable!("continue-as-new returns a control-flow error")
11206        });
11207
11208        let commands = worker
11209            .execute_workflow_task(workflow_task(
11210                "rust.typed-continue",
11211                Vec::new(),
11212                DEFAULT_CODEC,
11213            ))
11214            .expect("typed continue-as-new command");
11215
11216        assert_eq!(commands[0]["type"], "continue_as_new");
11217        assert_eq!(
11218            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
11219                .expect("typed continue arguments"),
11220            AvroValue::Array(vec![typed_fidelity_probe()])
11221        );
11222    }
11223
11224    #[test]
11225    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
11226        let client = Client::new("http://127.0.0.1:8080").expect("client");
11227        let mut worker = Worker::new(client, "rust-workers");
11228        worker.register_workflow("rust.continue", |ctx, _input| async move {
11229            ctx.continue_as_new(json!([2]))
11230        });
11231        let task = workflow_task(
11232            "rust.continue",
11233            vec![history_event(
11234                "WorkflowContinuedAsNew",
11235                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
11236            )],
11237            DEFAULT_CODEC,
11238        );
11239
11240        for _worker_restart_or_redelivery in 0..2 {
11241            let commands = worker
11242                .execute_workflow_task(task.clone())
11243                .expect("recorded transition replays");
11244            assert!(
11245                commands.is_empty(),
11246                "replay must not emit another successor"
11247            );
11248        }
11249    }
11250
11251    #[test]
11252    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
11253        let ctx = workflow_context(Vec::new());
11254        let error = ctx
11255            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
11256            .expect_err("blank queue must be rejected");
11257
11258        let Error::InvalidContinueAsNewOptions(error) = error else {
11259            panic!("expected typed continue-as-new validation error");
11260        };
11261        assert_eq!(error.field, "task_queue");
11262        assert!(ctx.take_commands().expect("commands").is_empty());
11263    }
11264
11265    #[test]
11266    fn workflow_context_exposes_server_history_budget() {
11267        let client = Client::new("http://127.0.0.1:8080").expect("client");
11268        let mut worker = Worker::new(client, "rust-workers");
11269        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
11270            let budget = ctx.history_budget()?;
11271            Ok(json!({
11272                "events": budget.event_count,
11273                "bytes": budget.size_bytes,
11274                "recommended": budget.continue_as_new_recommended,
11275                "pressure": budget.pressure,
11276            }))
11277        });
11278        let task: WorkflowTask = serde_json::from_value(json!({
11279            "task_id": "task-history-budget",
11280            "workflow_type": "rust.history-budget",
11281            "payload_codec": DEFAULT_CODEC,
11282            "history_events": [],
11283            "total_history_events": 480,
11284            "history_size_bytes": 1_048_576,
11285            "continue_as_new_recommended": true,
11286            "history_budget_pressure": "continue_as_new_recommended",
11287        }))
11288        .expect("published workflow task");
11289
11290        let commands = worker
11291            .execute_workflow_task(task)
11292            .expect("history-budget workflow");
11293        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
11294        assert_eq!(result["events"], 480);
11295        assert_eq!(result["bytes"], 1_048_576);
11296        assert_eq!(result["recommended"], true);
11297        assert_eq!(result["pressure"], "continue_as_new_recommended");
11298    }
11299
11300    #[test]
11301    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
11302        let client = Client::new("http://127.0.0.1:8080").expect("client");
11303        let mut worker = Worker::new(client, "rust-workers");
11304        worker.register_workflow("rust.failing", |_ctx, _input| async move {
11305            Err(Error::Codec("rust_conformance_failure".to_string()))
11306        });
11307        let task = WorkflowTask {
11308            task_id: "wft-rust-failing-1".to_string(),
11309            workflow_id: Some("wf-rust-failing".to_string()),
11310            run_id: Some("run-rust-failing".to_string()),
11311            workflow_type: "rust.failing".to_string(),
11312            payload_codec: DEFAULT_CODEC.to_string(),
11313            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11314            history_events: Vec::new(),
11315            total_history_events: Some(0),
11316            history_size_bytes: None,
11317            continue_as_new_recommended: None,
11318            history_budget_pressure: None,
11319            next_history_page_token: None,
11320            workflow_task_attempt: 1,
11321            workflow_signal_id: None,
11322            signal_name: None,
11323            signal_arguments: None,
11324            workflow_update_id: None,
11325            update_name: None,
11326            lease_owner: Some("rust-worker".to_string()),
11327        };
11328
11329        let commands = worker
11330            .execute_workflow_task(task)
11331            .expect("handler failure becomes a workflow command");
11332
11333        assert_eq!(commands.len(), 1);
11334        assert_eq!(commands[0]["type"], "fail_workflow");
11335        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
11336        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
11337        assert_eq!(commands[0]["non_retryable"], false);
11338        assert_eq!(
11339            commands[0]["message"],
11340            "codec error: rust_conformance_failure"
11341        );
11342        assert_eq!(
11343            commands[0]["exception"]["message"],
11344            "codec error: rust_conformance_failure"
11345        );
11346    }
11347
11348    #[test]
11349    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
11350        let client = Client::new("http://127.0.0.1:8080").expect("client");
11351        let mut worker = Worker::new(client, "rust-workers");
11352        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
11353            let _: String = ctx.side_effect(|| "captured".to_string())?;
11354            Err(Error::WorkerLoop("application failure".to_string()))
11355        });
11356
11357        let commands = worker
11358            .execute_workflow_task(workflow_task(
11359                "rust.failing-after-side-effect",
11360                Vec::new(),
11361                DEFAULT_CODEC,
11362            ))
11363            .expect("ordinary failure remains a workflow decision");
11364
11365        assert_eq!(commands.len(), 2);
11366        assert_eq!(commands[0]["type"], "record_side_effect");
11367        assert_eq!(commands[1]["type"], "fail_workflow");
11368    }
11369
11370    #[test]
11371    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
11372        let client = Client::new("http://127.0.0.1:8080").expect("client");
11373        let mut worker = Worker::new(client, "rust-workers");
11374        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
11375            Err(Error::WorkerLoop("application failure".to_string()))
11376        });
11377        let result =
11378            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
11379
11380        let error = worker
11381            .execute_workflow_task(workflow_task(
11382                "rust.removed-side-effect",
11383                vec![history_event(
11384                    "SideEffectRecorded",
11385                    json!({"sequence": 1, "result": result}),
11386                )],
11387                DEFAULT_CODEC,
11388            ))
11389            .expect_err("removed committed history must not become fail_workflow");
11390
11391        let Error::NonDeterministicReplay(failure) = error else {
11392            panic!("expected typed replay failure");
11393        };
11394        assert_eq!(failure.reason, "recorded_commands_unconsumed");
11395        assert_eq!(failure.sequence, Some(1));
11396        assert_eq!(failure.expected.as_deref(), Some("side effect"));
11397    }
11398
11399    #[test]
11400    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
11401        let client = Client::new("http://127.0.0.1:8080").expect("client");
11402        let mut worker = Worker::new(client, "rust-workers");
11403        worker.register_workflow(
11404            "rust.side-effect-before-marker-error",
11405            |ctx, _input| async move {
11406                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
11407                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
11408                ctx.get_version("restart-safe", 2, 2)?;
11409                Ok(Value::Null)
11410            },
11411        );
11412
11413        let error = worker
11414            .execute_workflow_task(workflow_task(
11415                "rust.side-effect-before-marker-error",
11416                vec![history_event(
11417                    "VersionMarkerRecorded",
11418                    json!({
11419                        "sequence": 1,
11420                        "change_id": "restart-safe",
11421                        "version": 1,
11422                        "min_supported": 1,
11423                        "max_supported": 1,
11424                    }),
11425                )],
11426                DEFAULT_CODEC,
11427            ))
11428            .expect_err("replay error must return no queued workflow commands");
11429
11430        let Error::NonDeterministicReplay(failure) = error else {
11431            panic!("expected typed replay failure");
11432        };
11433        assert_eq!(failure.reason, "version_marker_incompatible_range");
11434        assert_eq!(failure.sequence, Some(1));
11435    }
11436
11437    #[test]
11438    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
11439        let client = Client::new("http://127.0.0.1:8080").expect("client");
11440        let mut worker = Worker::new(client, "rust-workers");
11441        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
11442            ctx.sleep(Duration::from_secs(5)).await?;
11443            Ok(json!({"status": "timer fired"}))
11444        });
11445
11446        let task = WorkflowTask {
11447            task_id: "wft-rust-timer-pending".to_string(),
11448            workflow_id: Some("wf-rust-timer".to_string()),
11449            run_id: Some("run-rust-timer".to_string()),
11450            workflow_type: "rust.timer.pending".to_string(),
11451            payload_codec: DEFAULT_CODEC.to_string(),
11452            arguments: Some(
11453                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11454            ),
11455            history_events: vec![history_event(
11456                "TimerScheduled",
11457                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11458            )],
11459            total_history_events: Some(1),
11460            history_size_bytes: None,
11461            continue_as_new_recommended: None,
11462            history_budget_pressure: None,
11463            next_history_page_token: None,
11464            workflow_task_attempt: 1,
11465            workflow_signal_id: None,
11466            signal_name: None,
11467            signal_arguments: None,
11468            workflow_update_id: None,
11469            update_name: None,
11470            lease_owner: Some("rust-worker".to_string()),
11471        };
11472
11473        for _redelivery_or_restart in 0..2 {
11474            let commands = worker
11475                .execute_workflow_task(task.clone())
11476                .expect("recorded timer remains pending");
11477            assert!(
11478                commands.is_empty(),
11479                "recorded timer must not be rescheduled"
11480            );
11481        }
11482    }
11483
11484    #[test]
11485    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
11486        let client = Client::new("http://127.0.0.1:8080").expect("client");
11487        let mut worker = Worker::new(client, "rust-workers");
11488        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
11489            Ok(json!({"status": "completed"}))
11490        });
11491        let task = WorkflowTask {
11492            task_id: "wft-rust-timer-removed".to_string(),
11493            workflow_id: Some("wf-rust-timer".to_string()),
11494            run_id: Some("run-rust-timer".to_string()),
11495            workflow_type: "rust.timer.removed".to_string(),
11496            payload_codec: DEFAULT_CODEC.to_string(),
11497            arguments: Some(
11498                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11499            ),
11500            history_events: vec![
11501                history_event(
11502                    "TimerScheduled",
11503                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11504                ),
11505                history_event(
11506                    "TimerFired",
11507                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11508                ),
11509            ],
11510            total_history_events: Some(2),
11511            history_size_bytes: None,
11512            continue_as_new_recommended: None,
11513            history_budget_pressure: None,
11514            next_history_page_token: None,
11515            workflow_task_attempt: 1,
11516            workflow_signal_id: None,
11517            signal_name: None,
11518            signal_arguments: None,
11519            workflow_update_id: None,
11520            update_name: None,
11521            lease_owner: Some("rust-worker".to_string()),
11522        };
11523
11524        let Error::NonDeterministicReplay(failure) = worker
11525            .execute_workflow_task(task)
11526            .expect_err("removed timer must fail replay")
11527        else {
11528            panic!("expected typed replay failure");
11529        };
11530        assert_eq!(failure.reason, "recorded_commands_unconsumed");
11531        assert_eq!(failure.sequence, Some(1));
11532    }
11533
11534    #[test]
11535    fn workflow_context_emits_explicit_child_workflow_contract() {
11536        let ctx = WorkflowContext {
11537            state: Arc::new(Mutex::new(
11538                WorkflowState::new_with_identity(
11539                    Vec::new(),
11540                    Some("wf-parent".to_string()),
11541                    Some("run-parent".to_string()),
11542                    "parent-workers".to_string(),
11543                    DEFAULT_CODEC.to_string(),
11544                    None,
11545                )
11546                .expect("workflow state"),
11547            )),
11548        };
11549        let options = ChildWorkflowOptions::new("python-workers")
11550            .parent_close_policy(ParentClosePolicy::RequestCancel)
11551            .retry_policy(ChildWorkflowRetryPolicy {
11552                max_attempts: Some(3),
11553                backoff_seconds: vec![1, 5],
11554                non_retryable_error_types: vec!["ValidationError".to_string()],
11555            })
11556            .execution_timeout_seconds(600)
11557            .run_timeout_seconds(120);
11558        let mut call = Box::pin(ctx.start_child_workflow(
11559            "python.fulfil-order",
11560            options,
11561            json!([{"order_id": "order-42"}]),
11562        ));
11563        let mut task_context = TaskContext::from_waker(noop_waker_ref());
11564
11565        assert!(matches!(
11566            call.as_mut().poll(&mut task_context),
11567            Poll::Pending
11568        ));
11569        let commands = ctx.take_commands().expect("commands");
11570        assert_eq!(commands.len(), 1);
11571        let command = &commands[0];
11572        assert_eq!(command["type"], "start_child_workflow");
11573        assert_eq!(command["workflow_type"], "python.fulfil-order");
11574        assert_eq!(command["queue"], "python-workers");
11575        assert_eq!(command["parent_close_policy"], "request_cancel");
11576        assert_eq!(command["retry_policy"]["max_attempts"], 3);
11577        assert_eq!(command["execution_timeout_seconds"], 600);
11578        assert_eq!(command["run_timeout_seconds"], 120);
11579        assert_eq!(
11580            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
11581            json!([{"order_id": "order-42"}])
11582        );
11583    }
11584
11585    fn child_parent_worker() -> Worker {
11586        let client = Client::new("http://127.0.0.1:8080").expect("client");
11587        let mut worker = Worker::new(client, "rust-parent-workers");
11588        worker.register_workflow("rust.parent", |ctx, _input| async move {
11589            let child = ctx
11590                .start_child_workflow(
11591                    "python.child",
11592                    ChildWorkflowOptions::new("python-child-workers")
11593                        .parent_close_policy(ParentClosePolicy::Terminate),
11594                    json!([{"codec_probe": [1, true, "rust"]}]),
11595                )
11596                .await?;
11597            Ok(json!({
11598                "parent_workflow_id": child.parent.workflow_id,
11599                "parent_run_id": child.parent.run_id,
11600                "child_workflow_id": child.child.workflow_id,
11601                "child_run_id": child.child.run_id,
11602                "child_workflow_type": child.child_workflow_type,
11603                "result": child.result,
11604            }))
11605        });
11606        worker
11607    }
11608
11609    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11610        WorkflowTask {
11611            task_id: "wft-child-parent".to_string(),
11612            workflow_id: Some("wf-parent".to_string()),
11613            run_id: Some("run-parent".to_string()),
11614            workflow_type: "rust.parent".to_string(),
11615            payload_codec: DEFAULT_CODEC.to_string(),
11616            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11617            history_events: vec![
11618                HistoryEvent {
11619                    event_type: "ChildWorkflowScheduled".to_string(),
11620                    payload: json!({
11621                        "sequence": 1,
11622                        "child_call_id": "call-child",
11623                        "child_workflow_instance_id": "wf-child",
11624                        "child_workflow_run_id": "run-child",
11625                        "child_workflow_type": "python.child",
11626                    }),
11627                    raw: HashMap::new(),
11628                },
11629                HistoryEvent {
11630                    event_type: event_type.to_string(),
11631                    payload,
11632                    raw: HashMap::new(),
11633                },
11634            ],
11635            total_history_events: Some(2),
11636            history_size_bytes: None,
11637            continue_as_new_recommended: None,
11638            history_budget_pressure: None,
11639            next_history_page_token: None,
11640            workflow_task_attempt: 1,
11641            workflow_signal_id: None,
11642            signal_name: None,
11643            signal_arguments: None,
11644            workflow_update_id: None,
11645            update_name: None,
11646            lease_owner: Some("rust-worker".to_string()),
11647        }
11648    }
11649
11650    #[test]
11651    fn committed_child_result_replays_without_starting_a_duplicate() {
11652        let worker = child_parent_worker();
11653        let task = child_parent_task(
11654            "ChildRunCompleted",
11655            json!({
11656                "sequence": 1,
11657                "child_call_id": "call-child",
11658                "child_workflow_instance_id": "wf-child",
11659                "child_workflow_run_id": "run-child",
11660                "child_workflow_type": "python.child",
11661                "payload_codec": DEFAULT_CODEC,
11662                "result": fixture_envelope(json!({"from":"python","ok":true})),
11663            }),
11664        );
11665
11666        for _restart in 0..2 {
11667            let commands = worker
11668                .execute_workflow_task(task.clone())
11669                .expect("replayed parent task");
11670            assert_eq!(commands.len(), 1);
11671            assert_eq!(commands[0]["type"], "complete_workflow");
11672            assert!(!commands
11673                .iter()
11674                .any(|command| command["type"] == "start_child_workflow"));
11675            let output =
11676                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11677            assert_eq!(output["parent_workflow_id"], "wf-parent");
11678            assert_eq!(output["parent_run_id"], "run-parent");
11679            assert_eq!(output["child_workflow_id"], "wf-child");
11680            assert_eq!(output["child_run_id"], "run-child");
11681            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11682        }
11683    }
11684
11685    #[test]
11686    fn typed_child_arguments_and_results_survive_replay() {
11687        let client = Client::new("http://127.0.0.1:8080").expect("client");
11688        let mut worker = Worker::new(client, "rust-parent-workers");
11689        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11690            let child = ctx
11691                .start_child_workflow_avro_value(
11692                    "python.typed-child",
11693                    ChildWorkflowOptions::new("python-workers"),
11694                    AvroValue::Array(vec![typed_fidelity_probe()]),
11695                )
11696                .await?;
11697            Ok(child.result)
11698        });
11699
11700        let initial = worker
11701            .execute_workflow_task(workflow_task(
11702                "rust.typed-parent",
11703                Vec::new(),
11704                DEFAULT_CODEC,
11705            ))
11706            .expect("typed child start");
11707        assert_eq!(initial[0]["type"], "start_child_workflow");
11708        assert_eq!(
11709            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11710                .expect("typed child arguments"),
11711            AvroValue::Array(vec![typed_fidelity_probe()])
11712        );
11713
11714        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11715            .expect("typed child result");
11716        let task = workflow_task(
11717            "rust.typed-parent",
11718            vec![
11719                history_event(
11720                    "ChildWorkflowScheduled",
11721                    json!({
11722                        "sequence": 1,
11723                        "child_call_id": "call-typed",
11724                        "child_workflow_instance_id": "wf-child",
11725                        "child_workflow_run_id": "run-child",
11726                        "child_workflow_type": "python.typed-child",
11727                    }),
11728                ),
11729                history_event(
11730                    "ChildRunCompleted",
11731                    json!({
11732                        "sequence": 1,
11733                        "child_call_id": "call-typed",
11734                        "child_workflow_instance_id": "wf-child",
11735                        "child_workflow_run_id": "run-child",
11736                        "child_workflow_type": "python.typed-child",
11737                        "payload_codec": DEFAULT_CODEC,
11738                        "result": result,
11739                    }),
11740                ),
11741            ],
11742            DEFAULT_CODEC,
11743        );
11744
11745        let commands = worker
11746            .execute_workflow_task(task)
11747            .expect("typed child replay");
11748        assert_eq!(commands[0]["type"], "complete_workflow");
11749        assert_eq!(
11750            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11751                .expect("typed parent result"),
11752            typed_fidelity_probe()
11753        );
11754    }
11755
11756    #[test]
11757    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11758        let worker = child_parent_worker();
11759        let mut task = child_parent_task("unused", Value::Null);
11760        task.history_events.truncate(1);
11761        task.total_history_events = Some(1);
11762
11763        for _redelivery_or_restart in 0..2 {
11764            let commands = worker
11765                .execute_workflow_task(task.clone())
11766                .expect("recorded child remains pending");
11767            assert!(
11768                commands.is_empty(),
11769                "recorded pending child must not be started again"
11770            );
11771        }
11772    }
11773
11774    #[test]
11775    fn child_cancellation_becomes_stable_parent_failure_command() {
11776        let worker = child_parent_worker();
11777        let task = child_parent_task(
11778            "ChildRunCancelled",
11779            json!({
11780                "sequence": 1,
11781                "child_workflow_instance_id": "wf-child",
11782                "child_workflow_run_id": "run-child",
11783                "child_workflow_type": "python.child",
11784                "failure_id": "failure-child",
11785                "failure_category": "cancelled",
11786                "message": "cancelled by parent-close policy",
11787            }),
11788        );
11789
11790        let commands = worker
11791            .execute_workflow_task(task)
11792            .expect("parent settlement");
11793        assert_eq!(commands.len(), 1);
11794        assert_eq!(commands[0]["type"], "fail_workflow");
11795        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11796        assert_eq!(
11797            commands[0]["exception"]["properties"]["reason"],
11798            "cancelled"
11799        );
11800        assert_eq!(
11801            commands[0]["exception"]["properties"]["child_workflow_run_id"],
11802            "run-child"
11803        );
11804    }
11805
11806    #[test]
11807    fn workflow_can_handle_typed_child_failure() {
11808        let client = Client::new("http://127.0.0.1:8080").expect("client");
11809        let mut worker = Worker::new(client, "rust-parent-workers");
11810        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11811            match ctx
11812                .start_child_workflow(
11813                    "python.child",
11814                    ChildWorkflowOptions::new("python-child-workers"),
11815                    json!([]),
11816                )
11817                .await
11818            {
11819                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11820                    "reason": failure.reason,
11821                    "failure_id": failure.failure_id,
11822                    "exception_class": failure.exception_class,
11823                    "child_run_id": failure.child_workflow_run_id,
11824                })),
11825                Err(error) => Err(error),
11826                Ok(_) => Err(Error::WorkerLoop(
11827                    "child unexpectedly succeeded".to_string(),
11828                )),
11829            }
11830        });
11831        let mut task = child_parent_task(
11832            "ChildRunFailed",
11833            json!({
11834                "sequence": 1,
11835                "child_workflow_instance_id": "wf-child",
11836                "child_workflow_run_id": "run-child",
11837                "child_workflow_type": "python.child",
11838                "failure_id": "failure-child",
11839                "failure_category": "child_workflow",
11840                "message": "payment rejected",
11841                "exception": {
11842                    "type": "PaymentRejected",
11843                    "class": "payments.PaymentRejected",
11844                    "message": "payment rejected"
11845                }
11846            }),
11847        );
11848        task.workflow_type = "rust.handled-parent".to_string();
11849
11850        let commands = worker.execute_workflow_task(task).expect("handled failure");
11851        assert_eq!(commands[0]["type"], "complete_workflow");
11852        let output =
11853            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11854        assert_eq!(output["reason"], "child_workflow");
11855        assert_eq!(output["failure_id"], "failure-child");
11856        assert_eq!(output["exception_class"], "payments.PaymentRejected");
11857        assert_eq!(output["child_run_id"], "run-child");
11858    }
11859
11860    #[test]
11861    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11862        let client = Client::new("http://127.0.0.1:8080").expect("client");
11863        let mut worker = Worker::new(client, "rust-workers");
11864
11865        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11866            let signal = ctx.wait_signal("start").await?;
11867            let name = signal
11868                .first()
11869                .and_then(|value| value.as_str())
11870                .unwrap_or("world");
11871            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11872            Ok(json!({
11873                "greeting": greeting,
11874                "language": "rust"
11875            }))
11876        });
11877
11878        let signal_arguments =
11879            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11880        let task = WorkflowTask {
11881            task_id: "wft-rust-signal-1".to_string(),
11882            workflow_id: Some("wf-rust-hello".to_string()),
11883            run_id: Some("run-rust-hello".to_string()),
11884            workflow_type: "rust.hello_workflow".to_string(),
11885            payload_codec: DEFAULT_CODEC.to_string(),
11886            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11887            history_events: vec![HistoryEvent {
11888                event_type: "SignalReceived".to_string(),
11889                payload: json!({
11890                    "signal_id": "sig-rust-1",
11891                    "signal_name": "start"
11892                }),
11893                raw: HashMap::new(),
11894            }],
11895            total_history_events: Some(1),
11896            history_size_bytes: None,
11897            continue_as_new_recommended: None,
11898            history_budget_pressure: None,
11899            next_history_page_token: None,
11900            workflow_task_attempt: 1,
11901            workflow_signal_id: Some("sig-rust-1".to_string()),
11902            signal_name: Some("start".to_string()),
11903            signal_arguments: Some(signal_arguments),
11904            workflow_update_id: None,
11905            update_name: None,
11906            lease_owner: Some("rust-worker".to_string()),
11907        };
11908
11909        let commands = worker.execute_workflow_task(task).expect("workflow task");
11910
11911        assert_eq!(commands.len(), 1);
11912        assert_eq!(commands[0]["type"], "schedule_activity");
11913        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11914        assert_eq!(
11915            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11916            json!(["Rust"])
11917        );
11918    }
11919
11920    #[test]
11921    fn workflow_task_appends_paginated_history_events() {
11922        let mut task = WorkflowTask {
11923            task_id: "wft-rust-pages-1".to_string(),
11924            workflow_id: Some("wf-rust-pages".to_string()),
11925            run_id: Some("run-rust-pages".to_string()),
11926            workflow_type: "rust.hello_workflow".to_string(),
11927            payload_codec: DEFAULT_CODEC.to_string(),
11928            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11929            history_events: vec![HistoryEvent {
11930                event_type: "WorkflowStarted".to_string(),
11931                payload: json!({}),
11932                raw: HashMap::new(),
11933            }],
11934            total_history_events: Some(3),
11935            history_size_bytes: None,
11936            continue_as_new_recommended: None,
11937            history_budget_pressure: None,
11938            next_history_page_token: Some("MQ==".to_string()),
11939            workflow_task_attempt: 1,
11940            workflow_signal_id: None,
11941            signal_name: None,
11942            signal_arguments: None,
11943            workflow_update_id: None,
11944            update_name: None,
11945            lease_owner: Some("rust-worker".to_string()),
11946        };
11947
11948        task.append_history_page(WorkflowTaskHistoryPage {
11949            history_events: vec![
11950                HistoryEvent {
11951                    event_type: "SignalReceived".to_string(),
11952                    payload: json!({
11953                        "signal_id": "sig-rust-1",
11954                        "signal_name": "start",
11955                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11956                            .expect("signal arguments")
11957                    }),
11958                    raw: HashMap::new(),
11959                },
11960                HistoryEvent {
11961                    event_type: "MarkerRecorded".to_string(),
11962                    payload: json!({"sequence": 3}),
11963                    raw: HashMap::new(),
11964                },
11965            ],
11966            total_history_events: Some(3),
11967            next_history_page_token: None,
11968        });
11969
11970        assert_eq!(task.history_events.len(), 3);
11971        assert_eq!(task.total_history_events, Some(3));
11972        assert_eq!(task.next_history_page_token, None);
11973
11974        let signal = task
11975            .history_events
11976            .iter()
11977            .find(|event| event.event_type == "SignalReceived")
11978            .expect("signal event");
11979        assert_eq!(
11980            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11981            vec![AvroValue::String("Rust".to_string())]
11982        );
11983    }
11984
11985    #[tokio::test]
11986    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11987        let client = Client::new("http://127.0.0.1:8080").expect("client");
11988        let mut worker = Worker::new(client, "rust-workers");
11989        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11990        worker.register_query("counter", "current", |ctx, _args| async move {
11991            let mut count = 0_i64;
11992            for signal in ctx.signal_events() {
11993                let value = signal
11994                    .arguments
11995                    .first()
11996                    .and_then(Value::as_i64)
11997                    .unwrap_or_default();
11998                match signal.name.as_str() {
11999                    "increment" => count += value,
12000                    "set" => count = value,
12001                    _ => {}
12002                }
12003            }
12004            Ok(json!(count))
12005        });
12006
12007        let task = QueryTask {
12008            query_task_id: "query-rust-counter".to_string(),
12009            query_task_attempt: 1,
12010            lease_owner: Some("rust-worker".to_string()),
12011            workflow_id: Some("counter-1".to_string()),
12012            run_id: Some("run-counter-1".to_string()),
12013            workflow_type: "counter".to_string(),
12014            query_name: "current".to_string(),
12015            payload_codec: DEFAULT_CODEC.to_string(),
12016            workflow_arguments: Some(
12017                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
12018            ),
12019            query_arguments: Some(
12020                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
12021            ),
12022            history_events: vec![
12023                HistoryEvent {
12024                    event_type: "SignalReceived".to_string(),
12025                    payload: json!({
12026                        "signal_id": "php-signal-1",
12027                        "signal_name": "increment",
12028                        "workflow_sequence": 1,
12029                        "payload_codec": DEFAULT_CODEC,
12030                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
12031                    }),
12032                    raw: HashMap::new(),
12033                },
12034                HistoryEvent {
12035                    event_type: "SignalReceived".to_string(),
12036                    payload: json!({
12037                        "signal_id": "python-signal-2",
12038                        "signal_name": "increment",
12039                        "workflow_sequence": 2,
12040                        "payload_codec": DEFAULT_CODEC,
12041                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
12042                    }),
12043                    raw: HashMap::new(),
12044                },
12045                HistoryEvent {
12046                    event_type: "SignalReceived".to_string(),
12047                    payload: json!({
12048                        "signal_id": "rust-signal-3",
12049                        "signal_name": "set",
12050                        "workflow_sequence": 3,
12051                        "payload_codec": DEFAULT_CODEC,
12052                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
12053                    }),
12054                    raw: HashMap::new(),
12055                },
12056            ],
12057            history_export: None,
12058            run_status: Some("completed".to_string()),
12059        };
12060
12061        let result = worker.execute_query_task(task).await.expect("query result");
12062        assert_eq!(result.into_json().expect("query projection"), json!(0));
12063    }
12064
12065    #[tokio::test]
12066    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
12067        let worker = replay_counter_worker();
12068        let running_history = json!([
12069            {
12070                "type": "ActivityCompleted",
12071                "payload": {
12072                    "sequence": 1,
12073                    "activity_type": "load-counter",
12074                    "payload_codec": DEFAULT_CODEC,
12075                    "result": fixture_envelope(json!("loaded"))
12076                }
12077            },
12078            {
12079                "type": "SignalWaitOpened",
12080                "payload": {
12081                    "sequence": 3,
12082                    "signal_name": "increment"
12083                }
12084            },
12085            {
12086                "type": "SignalReceived",
12087                "payload": {
12088                    "signal_id": "signal-3",
12089                    "signal_name": "increment",
12090                    "workflow_sequence": 2,
12091                    "payload_codec": DEFAULT_CODEC,
12092                    "arguments": fixture_envelope(json!([3]))
12093                }
12094            },
12095            {
12096                "type": "SignalApplied",
12097                "payload": {
12098                    "sequence": 3,
12099                    "signal_id": "signal-3",
12100                    "signal_name": "increment",
12101                    "payload_codec": DEFAULT_CODEC,
12102                    "value": fixture_envelope(json!([3]))
12103                }
12104            }
12105        ]);
12106
12107        let running = worker
12108            .execute_query_task(replay_counter_query(
12109                "current",
12110                running_history.clone(),
12111                "running",
12112            ))
12113            .await
12114            .expect("running replay query");
12115        assert_eq!(
12116            running.clone().into_json().expect("query projection"),
12117            json!({"loaded": "loaded", "count": 3, "finished": false})
12118        );
12119
12120        let detached = worker
12121            .execute_query_task(replay_counter_query(
12122                "detached-mutation",
12123                running_history.clone(),
12124                "running",
12125            ))
12126            .await
12127            .expect("query mutates only its detached state clone");
12128        assert_eq!(detached.into_json().expect("query projection"), json!(999));
12129        let failed = worker
12130            .execute_query_task(replay_counter_query(
12131                "failed-mutation",
12132                running_history.clone(),
12133                "running",
12134            ))
12135            .await
12136            .expect_err("failed query");
12137        assert_eq!(failed.reason, "query_rejected");
12138        let unchanged = worker
12139            .execute_query_task(replay_counter_query("current", running_history, "running"))
12140            .await
12141            .expect("later query reconstructs unchanged state");
12142        assert_eq!(unchanged, running);
12143
12144        let restarted_worker = replay_counter_worker();
12145        let empty_arguments = fixture_envelope(json!([]));
12146        let loaded_result = fixture_envelope(json!("loaded"));
12147        let signal_three = fixture_blob(json!([3]));
12148        let signal_five = fixture_blob(json!([5]));
12149        let restarted_task: QueryTask = serde_json::from_value(json!({
12150            "query_task_id": "query-after-restart",
12151            "workflow_id": "counter-1",
12152            "run_id": "run-counter-1",
12153            "workflow_type": "replay-counter",
12154            "query_name": "current",
12155            "payload_codec": DEFAULT_CODEC,
12156            "workflow_arguments": empty_arguments.clone(),
12157            "query_arguments": empty_arguments,
12158            "history_events": [],
12159            "history_export": {
12160                "payloads": {"codec": DEFAULT_CODEC},
12161                "history_events": [
12162                    {
12163                        "type": "ActivityCompleted",
12164                        "payload": {
12165                            "sequence": 1,
12166                            "activity_type": "load-counter",
12167                            "payload_codec": DEFAULT_CODEC,
12168                            "result": null
12169                        }
12170                    },
12171                    {
12172                        "type": "SignalWaitOpened",
12173                        "payload": {
12174                            "sequence": 3,
12175                            "signal_name": "increment"
12176                        }
12177                    },
12178                    {
12179                        "type": "SignalReceived",
12180                        "payload": {
12181                            "signal_id": "signal-3",
12182                            "signal_name": "increment",
12183                            "workflow_sequence": 2
12184                        }
12185                    },
12186                    {
12187                        "type": "SignalApplied",
12188                        "payload": {
12189                            "sequence": 3,
12190                            "signal_id": "signal-3",
12191                            "signal_name": "increment"
12192                        }
12193                    },
12194                    {
12195                        "type": "SignalWaitOpened",
12196                        "payload": {
12197                            "sequence": 5,
12198                            "signal_name": "increment"
12199                        }
12200                    },
12201                    {
12202                        "type": "SignalReceived",
12203                        "payload": {
12204                            "signal_id": "signal-5",
12205                            "signal_name": "increment",
12206                            "workflow_sequence": 4
12207                        }
12208                    },
12209                    {
12210                        "type": "SignalApplied",
12211                        "payload": {
12212                            "sequence": 5,
12213                            "signal_id": "signal-5",
12214                            "signal_name": "increment"
12215                        }
12216                    }
12217                ],
12218                "activities": [{
12219                    "sequence": 1,
12220                    "activity_type": "load-counter",
12221                    "payload_codec": DEFAULT_CODEC,
12222                    "result": loaded_result
12223                }],
12224                "signals": [
12225                    {
12226                        "id": "signal-3",
12227                        "name": "increment",
12228                        "workflow_sequence": 2,
12229                        "payload_codec": DEFAULT_CODEC,
12230                        "arguments": signal_three
12231                    },
12232                    {
12233                        "id": "signal-5",
12234                        "name": "increment",
12235                        "workflow_sequence": 4,
12236                        "payload_codec": DEFAULT_CODEC,
12237                        "arguments": signal_five
12238                    }
12239                ]
12240            },
12241            "run_status": "completed"
12242        }))
12243        .expect("cold replay query task");
12244        let completed = restarted_worker
12245            .execute_query_task(restarted_task)
12246            .await
12247            .expect("completed cold replay query");
12248        assert_eq!(
12249            completed.into_json().expect("query projection"),
12250            json!({"loaded": "loaded", "count": 8, "finished": true})
12251        );
12252    }
12253
12254    #[tokio::test]
12255    async fn replayed_query_replay_failures_are_machine_readable() {
12256        let worker = replay_counter_worker();
12257        let task = replay_counter_query(
12258            "current",
12259            json!([{
12260                "type": "ActivityCompleted",
12261                "payload": {
12262                    "sequence": 1,
12263                    "payload_codec": DEFAULT_CODEC,
12264                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
12265                }
12266            }]),
12267            "running",
12268        );
12269        let failure = worker
12270            .execute_query_task(task)
12271            .await
12272            .expect_err("invalid replay history payload");
12273        assert_eq!(failure.reason, "query_payload_decode_failed");
12274        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
12275        assert!(failure.message.contains("invalid_payload_framing"));
12276    }
12277
12278    #[tokio::test]
12279    async fn query_task_restores_compact_history_from_export() {
12280        let client = Client::new("http://127.0.0.1:8080").expect("client");
12281        let mut worker = Worker::new(client, "rust-workers");
12282        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12283        worker.register_query("counter", "current", |ctx, _args| async move {
12284            Ok(json!(ctx.signals("increment")[0][0]))
12285        });
12286        let empty_arguments = fixture_envelope(json!([]));
12287        let exported_signal = fixture_blob(json!([9]));
12288        let task: QueryTask = serde_json::from_value(json!({
12289            "query_task_id": "query-export",
12290            "workflow_type": "counter",
12291            "query_name": "current",
12292            "payload_codec": DEFAULT_CODEC,
12293            "workflow_arguments": empty_arguments.clone(),
12294            "query_arguments": empty_arguments,
12295            "history_events": [],
12296            "history_export": {
12297                "payloads": {"codec": DEFAULT_CODEC},
12298                "history_events": [{
12299                    "type": "SignalReceived",
12300                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
12301                }],
12302                "signals": [{
12303                    "id": "signal-export",
12304                    "name": "increment",
12305                    "status": "applied",
12306                    "workflow_sequence": 1,
12307                    "payload_codec": DEFAULT_CODEC,
12308                    "arguments": exported_signal
12309                }]
12310            }
12311        }))
12312        .expect("query task");
12313
12314        let result = worker.execute_query_task(task).await.expect("query result");
12315        assert_eq!(result.into_json().expect("query projection"), json!(9));
12316    }
12317
12318    #[tokio::test]
12319    async fn query_task_failures_have_stable_reasons() {
12320        let client = Client::new("http://127.0.0.1:8080").expect("client");
12321        let mut worker = Worker::new(client, "rust-workers");
12322        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12323        worker.register_query(
12324            "counter",
12325            "current",
12326            |_ctx, _args| async move { Ok(json!(0)) },
12327        );
12328
12329        let base_task = QueryTask {
12330            query_task_id: "query-errors".to_string(),
12331            query_task_attempt: 1,
12332            lease_owner: None,
12333            workflow_id: Some("counter-errors".to_string()),
12334            run_id: Some("run-errors".to_string()),
12335            workflow_type: "counter".to_string(),
12336            query_name: "missing".to_string(),
12337            payload_codec: DEFAULT_CODEC.to_string(),
12338            workflow_arguments: Some(fixture_envelope(json!([]))),
12339            query_arguments: Some(fixture_envelope(json!([]))),
12340            history_events: Vec::new(),
12341            history_export: None,
12342            run_status: Some("running".to_string()),
12343        };
12344
12345        let unknown = worker
12346            .execute_query_task(base_task.clone())
12347            .await
12348            .expect_err("unknown query");
12349        assert_eq!(unknown.reason, "rejected_unknown_query");
12350
12351        let mut malformed = base_task;
12352        malformed.query_name = "current".to_string();
12353        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
12354        let malformed = worker
12355            .execute_query_task(malformed)
12356            .await
12357            .expect_err("malformed payload");
12358        assert_eq!(malformed.reason, "query_payload_decode_failed");
12359
12360        let client = Client::new("http://127.0.0.1:8080").expect("client");
12361        let mut unavailable_worker = Worker::new(client, "rust-workers");
12362        unavailable_worker
12363            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12364        let empty_arguments = fixture_envelope(json!([]));
12365        let unavailable_task: QueryTask = serde_json::from_value(json!({
12366            "query_task_id": "query-unavailable",
12367            "workflow_type": "counter",
12368            "query_name": "current",
12369            "payload_codec": DEFAULT_CODEC,
12370            "workflow_arguments": empty_arguments.clone(),
12371            "query_arguments": empty_arguments
12372        }))
12373        .expect("query task");
12374        let unavailable = unavailable_worker
12375            .execute_query_task(unavailable_task)
12376            .await
12377            .expect_err("query handler unavailable");
12378        assert_eq!(unavailable.reason, "query_handler_unavailable");
12379    }
12380
12381    #[tokio::test]
12382    async fn client_query_decodes_result_and_typed_failure() {
12383        let server = MockWorkerServer::start();
12384        let client = Client::builder(server.base_url())
12385            .timeout(Duration::from_secs(2))
12386            .build()
12387            .expect("client");
12388
12389        let result = client
12390            .query_workflow("counter-1", "current", json!([]))
12391            .await
12392            .expect("query result");
12393        assert_eq!(result, json!({"count": 8}));
12394
12395        let error = client
12396            .query_workflow("counter-1", "missing", json!([]))
12397            .await
12398            .expect_err("unknown query");
12399        let Error::QueryFailed(failure) = error else {
12400            panic!("expected typed query failure");
12401        };
12402        assert_eq!(failure.status, 404);
12403        assert_eq!(failure.reason, "rejected_unknown_query");
12404    }
12405
12406    #[tokio::test]
12407    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
12408        let server = MockWorkerServer::start();
12409        let client = Client::builder(server.base_url())
12410            .timeout(Duration::from_secs(2))
12411            .build()
12412            .expect("client");
12413        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
12414
12415        client
12416            .start_workflow(
12417                "typed.echo",
12418                "rust-workers",
12419                "typed-start",
12420                arguments.clone(),
12421            )
12422            .await
12423            .expect("typed workflow start");
12424        assert_eq!(
12425            decode_wire_avro_value(
12426                &server.request_body("/api/workflows")["input"],
12427                DEFAULT_CODEC,
12428            )
12429            .expect("typed start input"),
12430            arguments
12431        );
12432
12433        client
12434            .signal_workflow("typed-1", "changed", arguments.clone())
12435            .await
12436            .expect("typed signal");
12437        assert_eq!(
12438            decode_wire_avro_value(
12439                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
12440                DEFAULT_CODEC,
12441            )
12442            .expect("typed signal input"),
12443            arguments
12444        );
12445
12446        assert_eq!(
12447            client
12448                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
12449                .await
12450                .expect("typed query"),
12451            typed_fidelity_probe()
12452        );
12453        assert_eq!(
12454            decode_wire_avro_value(
12455                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
12456                DEFAULT_CODEC,
12457            )
12458            .expect("typed query input"),
12459            arguments
12460        );
12461
12462        assert_eq!(
12463            client
12464                .update_workflow_avro_value(
12465                    "typed-1",
12466                    "replace",
12467                    arguments.clone(),
12468                    Some("typed-request"),
12469                )
12470                .await
12471                .expect("typed update"),
12472            typed_fidelity_probe()
12473        );
12474        let update = server.request_body("/api/workflows/typed-1/update/replace");
12475        assert_eq!(update["request_id"], "typed-request");
12476        assert_eq!(
12477            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
12478            arguments
12479        );
12480
12481        let handle = WorkflowHandle {
12482            client: client.clone(),
12483            workflow_id: "typed-1".to_string(),
12484            run_id: Some("run-typed-1".to_string()),
12485            workflow_type: "typed.echo".to_string(),
12486        };
12487        assert_eq!(
12488            handle
12489                .result_avro_value(WorkflowResultOptions::default())
12490                .await
12491                .expect("typed workflow result"),
12492            typed_fidelity_probe()
12493        );
12494
12495        client
12496            .complete_activity_task(
12497                "activity-typed",
12498                "attempt-typed",
12499                "rust-worker",
12500                typed_fidelity_probe(),
12501                DEFAULT_CODEC,
12502            )
12503            .await
12504            .expect("typed activity completion");
12505        assert_eq!(
12506            decode_wire_avro_value(
12507                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
12508                    ["result"],
12509                DEFAULT_CODEC,
12510            )
12511            .expect("typed activity result"),
12512            typed_fidelity_probe()
12513        );
12514        client
12515            .fail_activity_task(
12516                "activity-typed",
12517                "attempt-typed",
12518                "rust-worker",
12519                "typed failure",
12520                true,
12521            )
12522            .await
12523            .expect("activity failure");
12524    }
12525
12526    #[tokio::test]
12527    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
12528        let server = MockWorkerServer::start();
12529        let client = Client::builder(server.base_url())
12530            .timeout(Duration::from_secs(2))
12531            .build()
12532            .expect("client");
12533
12534        let options = WorkflowCommandOptions::new()
12535            .reason("cleanup requested")
12536            .request_id("cancel-17");
12537        let cancelled = client
12538            .cancel_workflow("wf-lifecycle", options)
12539            .await
12540            .expect("instance cancellation");
12541        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
12542        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
12543        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
12544        assert_eq!(
12545            server.request_body("/api/workflows/wf-lifecycle/cancel"),
12546            json!({"reason":"cleanup requested","request_id":"cancel-17"})
12547        );
12548
12549        let terminated = client
12550            .terminate_workflow(
12551                "wf-lifecycle",
12552                WorkflowCommandOptions::new().reason("forced stop"),
12553            )
12554            .await
12555            .expect("instance termination");
12556        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
12557        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
12558
12559        client
12560            .cancel_workflow_run(
12561                "wf-lifecycle",
12562                "run-current",
12563                WorkflowCommandOptions::default(),
12564            )
12565            .await
12566            .expect("selected run cancellation");
12567        client
12568            .terminate_workflow_run(
12569                "wf-lifecycle",
12570                "run-current",
12571                WorkflowCommandOptions::default(),
12572            )
12573            .await
12574            .expect("selected run termination");
12575
12576        for (command, error) in [
12577            (
12578                WorkflowCommandKind::Cancel,
12579                client
12580                    .cancel_workflow_run(
12581                        "wf-lifecycle",
12582                        "run-stale",
12583                        WorkflowCommandOptions::default(),
12584                    )
12585                    .await
12586                    .expect_err("stale cancellation must be rejected"),
12587            ),
12588            (
12589                WorkflowCommandKind::Terminate,
12590                client
12591                    .terminate_workflow_run(
12592                        "wf-lifecycle",
12593                        "run-stale",
12594                        WorkflowCommandOptions::default(),
12595                    )
12596                    .await
12597                    .expect_err("stale termination must be rejected"),
12598            ),
12599        ] {
12600            let Error::WorkflowCommandRejected(rejection) = error else {
12601                panic!("expected typed command rejection");
12602            };
12603            assert_eq!(rejection.command, command);
12604            assert_eq!(rejection.status, 409);
12605            assert_eq!(rejection.reason, "historical_run_command_rejected");
12606            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12607            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12608        }
12609    }
12610
12611    #[tokio::test]
12612    async fn workflow_start_options_send_server_enforced_deadlines() {
12613        let server = MockWorkerServer::start();
12614        let client = Client::builder(server.base_url())
12615            .timeout(Duration::from_secs(2))
12616            .build()
12617            .expect("client");
12618
12619        let handle = client
12620            .start_workflow_with_options(
12621                "rust.timeout",
12622                "rust-timeouts",
12623                "wf-start-options",
12624                WorkflowStartOptions::new()
12625                    .execution_timeout_seconds(30)
12626                    .run_timeout_seconds(1),
12627                json!([]),
12628            )
12629            .await
12630            .expect("workflow start");
12631
12632        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12633        let body = server.request_body("/api/workflows");
12634        assert_eq!(body["execution_timeout_seconds"], 30);
12635        assert_eq!(body["run_timeout_seconds"], 1);
12636
12637        let invalid = client
12638            .start_workflow_with_options(
12639                "rust.timeout",
12640                "rust-timeouts",
12641                "wf-invalid-options",
12642                WorkflowStartOptions::new()
12643                    .execution_timeout_seconds(1)
12644                    .run_timeout_seconds(2),
12645                json!([]),
12646            )
12647            .await
12648            .expect_err("invalid deadline ordering");
12649        assert!(invalid
12650            .to_string()
12651            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12652    }
12653
12654    #[tokio::test]
12655    async fn workflow_result_returns_each_typed_terminal_outcome() {
12656        let server = MockWorkerServer::start();
12657        let client = Client::builder(server.base_url())
12658            .timeout(Duration::from_secs(2))
12659            .build()
12660            .expect("client");
12661        let options = WorkflowResultOptions {
12662            poll_interval: Duration::ZERO,
12663            timeout: Duration::from_secs(1),
12664        };
12665
12666        let failed = WorkflowHandle {
12667            client: client.clone(),
12668            workflow_id: "wf-failed".to_string(),
12669            run_id: Some("run-failed".to_string()),
12670            workflow_type: "failure".to_string(),
12671        }
12672        .result(options)
12673        .await
12674        .expect_err("failed outcome");
12675        let Error::WorkflowFailed(failure) = failed else {
12676            panic!("expected WorkflowFailed");
12677        };
12678        assert_eq!(failure.workflow_id, "wf-failed");
12679        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12680        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12681        assert_eq!(failure.failure_category.as_deref(), Some("application"));
12682        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12683        assert_eq!(
12684            failure.exception_class.as_deref(),
12685            Some("billing::PaymentError")
12686        );
12687        assert_eq!(failure.non_retryable, Some(true));
12688
12689        for (workflow_id, expected_kind, expected_reason) in [
12690            (
12691                "wf-cancelled",
12692                WorkflowTerminalKind::Cancelled,
12693                "cleanup requested",
12694            ),
12695            (
12696                "wf-terminated",
12697                WorkflowTerminalKind::Terminated,
12698                "forced stop",
12699            ),
12700            (
12701                "wf-timed-out",
12702                WorkflowTerminalKind::TimedOut,
12703                "run_timeout",
12704            ),
12705        ] {
12706            let error = WorkflowHandle {
12707                client: client.clone(),
12708                workflow_id: workflow_id.to_string(),
12709                run_id: None,
12710                workflow_type: "terminal".to_string(),
12711            }
12712            .result(options)
12713            .await
12714            .expect_err("typed terminal outcome");
12715            let outcome = match error {
12716                Error::WorkflowCancelled(outcome) => outcome,
12717                Error::WorkflowTerminated(outcome) => outcome,
12718                Error::WorkflowTimedOut(outcome) => outcome,
12719                other => panic!("unexpected terminal error: {other}"),
12720            };
12721            assert_eq!(outcome.kind, expected_kind);
12722            assert_eq!(outcome.workflow_id, workflow_id);
12723            assert_eq!(outcome.reason, expected_reason);
12724        }
12725
12726        let wait_timeout = WorkflowHandle {
12727            client,
12728            workflow_id: "wf-waiting".to_string(),
12729            run_id: Some("run-waiting".to_string()),
12730            workflow_type: "waiting".to_string(),
12731        }
12732        .result(WorkflowResultOptions {
12733            poll_interval: Duration::ZERO,
12734            timeout: Duration::ZERO,
12735        })
12736        .await
12737        .expect_err("client wait timeout");
12738        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12739            panic!("expected typed client timeout");
12740        };
12741        assert_eq!(timeout.reason, "result_wait_timeout");
12742        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12743        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12744    }
12745
12746    #[tokio::test]
12747    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12748        let server = MockWorkerServer::start();
12749        let client = Client::builder(server.base_url())
12750            .timeout(Duration::from_secs(2))
12751            .build()
12752            .expect("client");
12753
12754        let handle = WorkflowHandle {
12755            client,
12756            workflow_id: "wf-selected".to_string(),
12757            run_id: Some("run-selected".to_string()),
12758            workflow_type: "selected".to_string(),
12759        };
12760        let options = WorkflowResultOptions {
12761            poll_interval: Duration::ZERO,
12762            timeout: Duration::from_secs(1),
12763        };
12764
12765        let current = handle
12766            .result(options)
12767            .await
12768            .expect("instance result follows the current run");
12769        assert_eq!(current, json!("current run output"));
12770
12771        let error = handle
12772            .result_selected_run(options)
12773            .await
12774            .expect_err("the selected run is cancelled even though the current run completed");
12775
12776        let Error::WorkflowCancelled(outcome) = error else {
12777            panic!("expected selected run cancellation");
12778        };
12779        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12780        assert_eq!(outcome.reason, "selected run cancelled");
12781        assert_eq!(
12782            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12783            1
12784        );
12785        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12786    }
12787
12788    #[tokio::test]
12789    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12790        let server = MockWorkerServer::draining_polls();
12791        let client = Client::builder(server.base_url())
12792            .timeout(Duration::from_secs(2))
12793            .build()
12794            .expect("client");
12795
12796        let workflow = client
12797            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12798            .await
12799            .expect("workflow drain response");
12800        let activity = client
12801            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12802            .await
12803            .expect("activity drain response");
12804        let query = client
12805            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12806            .await
12807            .expect("query drain response");
12808
12809        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12810            assert_eq!(
12811                outcome,
12812                WorkerPollOutcome::Stop {
12813                    poll_status: Some("draining".to_string()),
12814                    reason: Some("worker_draining".to_string()),
12815                }
12816            );
12817        }
12818
12819        assert!(client
12820            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12821            .await
12822            .expect("compatibility poll")
12823            .is_none());
12824    }
12825
12826    #[tokio::test]
12827    async fn managed_worker_honors_drain_stop_for_every_task_family() {
12828        let server = MockWorkerServer::draining_polls();
12829        let client = Client::builder(server.base_url())
12830            .timeout(Duration::from_secs(2))
12831            .build()
12832            .expect("client");
12833
12834        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12835            .worker_id("draining-workflow-worker")
12836            .poll_timeout(Duration::ZERO);
12837        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12838        workflow_worker
12839            .run()
12840            .await
12841            .expect("workflow drain is a clean stop");
12842
12843        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12844            .worker_id("draining-activity-worker")
12845            .poll_timeout(Duration::ZERO);
12846        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12847        activity_worker
12848            .run()
12849            .await
12850            .expect("activity drain is a clean stop");
12851
12852        let mut query_worker = Worker::new(client, "rust-workers")
12853            .worker_id("draining-query-worker")
12854            .poll_timeout(Duration::ZERO);
12855        query_worker.register_query("counter", "current", |_ctx, _args| async {
12856            Ok(Value::Null)
12857        });
12858        query_worker
12859            .run()
12860            .await
12861            .expect("query drain is a clean stop");
12862    }
12863
12864    #[tokio::test]
12865    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12866        let server = MockWorkerServer::start();
12867        let client = Client::builder(server.base_url())
12868            .timeout(Duration::from_secs(2))
12869            .build()
12870            .expect("client");
12871
12872        let heartbeat = client
12873            .heartbeat_activity_task(
12874                "activity-cancel",
12875                "attempt-cancel",
12876                "rust-worker",
12877                typed_fidelity_probe(),
12878            )
12879            .await
12880            .expect("cancellation heartbeat");
12881        assert!(heartbeat.cancel_requested);
12882        assert!(heartbeat.should_stop());
12883        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12884        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12885        let heartbeat_body =
12886            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12887        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12888        assert_eq!(
12889            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12890                .expect("typed heartbeat details"),
12891            typed_fidelity_probe()
12892        );
12893
12894        let error = client
12895            .complete_activity_task(
12896                "activity-cancel",
12897                "attempt-cancel",
12898                "rust-worker",
12899                json!({"late":true}),
12900                DEFAULT_CODEC,
12901            )
12902            .await
12903            .expect_err("late completion must be refused");
12904        assert!(activity_task_rejection_is_final(&error));
12905        let Error::ActivityTaskRejected(rejection) = error else {
12906            panic!("expected typed activity rejection");
12907        };
12908        assert_eq!(rejection.status, 409);
12909        assert_eq!(rejection.reason, "run_cancelled");
12910        assert!(rejection.cancel_requested);
12911        assert_eq!(rejection.can_continue, Some(false));
12912    }
12913
12914    #[tokio::test]
12915    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12916        let server = MockWorkerServer::cancelled_activity();
12917        let client = Client::builder(server.base_url())
12918            .timeout(Duration::from_secs(2))
12919            .build()
12920            .expect("client");
12921        let cancellation_observed = Arc::new(AtomicBool::new(false));
12922        let observed = Arc::clone(&cancellation_observed);
12923        let mut worker = Worker::new(client.clone(), "rust-workers")
12924            .worker_id("rust-cancel-worker")
12925            .poll_timeout(Duration::from_millis(10));
12926        worker.register_activity("cancel-aware", move |ctx, _args| {
12927            let observed = Arc::clone(&observed);
12928            async move {
12929                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12930                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12931                Ok(json!({"late":"completion"}))
12932            }
12933        });
12934
12935        assert_eq!(
12936            worker.run_once().await.expect("cancelled attempt handled"),
12937            1
12938        );
12939        assert!(cancellation_observed.load(Ordering::SeqCst));
12940        assert_eq!(
12941            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12942            1
12943        );
12944
12945        let mut restarted = Worker::new(client, "rust-workers")
12946            .worker_id("rust-cancel-worker-restarted")
12947            .poll_timeout(Duration::from_millis(10));
12948        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12949        assert_eq!(
12950            restarted
12951                .run_once()
12952                .await
12953                .expect("replacement worker continues polling"),
12954            0
12955        );
12956    }
12957
12958    #[tokio::test]
12959    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12960        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12961        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12962        let client = Client::builder(server.base_url())
12963            .timeout(Duration::from_secs(2))
12964            .build()
12965            .expect("client");
12966
12967        let direct_error = client
12968            .complete_workflow_task(
12969                "workflow-timeout-task",
12970                "timeout-worker",
12971                3,
12972                vec![json!({
12973                    "type": "complete_workflow",
12974                    "result": fixture_envelope(Value::Null)
12975                })],
12976            )
12977            .await
12978            .expect_err("the low-level client preserves the completion rejection");
12979        let Error::Http { status, body } = direct_error else {
12980            panic!("expected the original HTTP completion rejection");
12981        };
12982        assert_eq!(status, reqwest::StatusCode::CONFLICT);
12983        assert_eq!(
12984            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12985            "run_timed_out"
12986        );
12987
12988        let mut worker = Worker::new(client, "rust-workers")
12989            .worker_id("timeout-worker")
12990            .poll_timeout(Duration::from_millis(10));
12991        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12992            Ok(json!({"late": "result"}))
12993        });
12994
12995        assert_eq!(
12996            worker
12997                .run_once()
12998                .await
12999                .expect("authoritative selected-run timeout settles the tick"),
13000            1
13001        );
13002        assert_eq!(
13003            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
13004            2,
13005            "both the direct client proof and managed worker must see the rejection"
13006        );
13007    }
13008
13009    #[tokio::test]
13010    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
13011        for (name, status, response) in [
13012            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
13013            (
13014                "command was recorded",
13015                "409 Conflict",
13016                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13017            ),
13018            (
13019                "lease conflict",
13020                "409 Conflict",
13021                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
13022            ),
13023            (
13024                "nonterminal run",
13025                "409 Conflict",
13026                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
13027            ),
13028            (
13029                "different selected run",
13030                "409 Conflict",
13031                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
13032            ),
13033            (
13034                "different task attempt",
13035                "409 Conflict",
13036                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13037            ),
13038            (
13039                "authentication failure",
13040                "401 Unauthorized",
13041                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13042            ),
13043            (
13044                "authorization failure",
13045                "403 Forbidden",
13046                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13047            ),
13048            (
13049                "protocol failure",
13050                "400 Bad Request",
13051                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
13052            ),
13053            (
13054                "malformed command",
13055                "422 Unprocessable Entity",
13056                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13057            ),
13058            (
13059                "transient server failure",
13060                "503 Service Unavailable",
13061                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13062            ),
13063        ] {
13064            let server = MockWorkerServer::workflow_completion(status, response);
13065            let client = Client::builder(server.base_url())
13066                .timeout(Duration::from_secs(2))
13067                .build()
13068                .expect("client");
13069            let mut worker = Worker::new(client, "rust-workers")
13070                .worker_id("timeout-worker")
13071                .poll_timeout(Duration::from_millis(10));
13072            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
13073                Ok(json!({"late": "result"}))
13074            });
13075
13076            let error = worker
13077                .run_once()
13078                .await
13079                .expect_err(&format!("{name} must remain an error"));
13080            assert!(
13081                matches!(error, Error::Http { .. } | Error::Protocol(_)),
13082                "{name} returned an unexpected error variant: {error}"
13083            );
13084        }
13085    }
13086
13087    #[tokio::test]
13088    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
13089        let server = MockWorkerServer::start();
13090        let client = Client::builder(server.base_url())
13091            .worker_token(Some("worker-secret".to_string()))
13092            .namespace("orders")
13093            .timeout(Duration::from_secs(2))
13094            .build()
13095            .expect("client");
13096        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
13097
13098        let result = client
13099            .deregister_worker_registration("worker/α space")
13100            .await
13101            .expect("deregister worker registration");
13102
13103        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
13104        assert_eq!(
13105            server.worker_protocol_for(path).as_deref(),
13106            Some(WORKER_PROTOCOL_VERSION)
13107        );
13108        assert_eq!(server.control_protocol_for(path), None);
13109        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
13110        assert_eq!(
13111            server.authorization_for(path).as_deref(),
13112            Some("Bearer worker-secret")
13113        );
13114        assert_eq!(
13115            result,
13116            WorkerDeregistrationEnvelope {
13117                worker_id: "deregistered-worker".to_string(),
13118                outcome: "deregistered".to_string(),
13119                recovered_workflow_task_count: 2,
13120            }
13121        );
13122    }
13123
13124    #[tokio::test]
13125    async fn low_level_registration_rejects_update_validators_before_transport() {
13126        let server = MockWorkerServer::start();
13127        let client = Client::builder(server.base_url())
13128            .timeout(Duration::from_secs(2))
13129            .build()
13130            .expect("client");
13131
13132        for update_validators in [json!(["approve"]), json!("approve")] {
13133            let error = client
13134                .register_worker_with_command_contracts(
13135                    "validator-claiming-worker",
13136                    "rust-workers",
13137                    vec!["orders".to_string()],
13138                    vec![],
13139                    1,
13140                    1,
13141                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13142                    json!({
13143                        "orders": {
13144                            "queries": ["current"],
13145                            "updates": ["approve"],
13146                            "update_validators": update_validators,
13147                        },
13148                    }),
13149                )
13150                .await
13151                .expect_err("unsupported validator claims must fail before registration");
13152
13153            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
13154                panic!("expected typed unsupported-validator failure");
13155            };
13156            assert_eq!(workflow_type, "orders");
13157        }
13158        assert_eq!(server.request_count("/api/worker/register"), 0);
13159    }
13160
13161    #[tokio::test]
13162    async fn low_level_registration_preserves_query_and_update_contracts() {
13163        let server = MockWorkerServer::start();
13164        let client = Client::builder(server.base_url())
13165            .timeout(Duration::from_secs(2))
13166            .build()
13167            .expect("client");
13168        let contracts = json!({
13169            "orders": {
13170                "queries": ["current"],
13171                "updates": ["approve"],
13172                "update_validators": [],
13173            },
13174            "payments": {
13175                "queries": ["status"],
13176                "updates": ["capture"],
13177            },
13178        });
13179
13180        client
13181            .register_worker_with_command_contracts(
13182                "command-worker",
13183                "rust-workers",
13184                vec!["orders".to_string(), "payments".to_string()],
13185                vec![],
13186                1,
13187                1,
13188                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13189                contracts.clone(),
13190            )
13191            .await
13192            .expect("query and update contracts must remain supported");
13193
13194        assert_eq!(
13195            server.request_body("/api/worker/register")["workflow_command_contracts"],
13196            contracts
13197        );
13198    }
13199
13200    #[tokio::test]
13201    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
13202        let server = MockWorkerServer::start();
13203        let control_only = Client::builder(server.base_url())
13204            .control_token(Some("control-secret".to_string()))
13205            .build()
13206            .expect("control client");
13207
13208        let error = control_only
13209            .register_worker("worker", "queue", vec![], vec![], 1, 1)
13210            .await
13211            .expect_err("control token must not authorize a worker request");
13212        assert!(matches!(
13213            error,
13214            Error::MissingRoleCredentials { role: "worker", .. }
13215        ));
13216        assert_eq!(server.request_count("/api/worker/register"), 0);
13217
13218        let worker_only = Client::builder(server.base_url())
13219            .worker_token(Some("worker-secret".to_string()))
13220            .build()
13221            .expect("worker client");
13222        let error = worker_only
13223            .health()
13224            .await
13225            .expect_err("worker token must not authorize a control request");
13226        assert!(matches!(
13227            error,
13228            Error::MissingRoleCredentials {
13229                role: "control",
13230                ..
13231            }
13232        ));
13233        assert_eq!(server.request_count("/api/health"), 0);
13234    }
13235
13236    #[tokio::test]
13237    async fn shared_token_supports_worker_and_control_planes() {
13238        let server = MockWorkerServer::start();
13239        let client = Client::builder(server.base_url())
13240            .token(Some("shared-secret".to_string()))
13241            .build()
13242            .expect("client");
13243
13244        client.health().await.expect("control request");
13245        client
13246            .register_worker("worker", "queue", vec![], vec![], 1, 1)
13247            .await
13248            .expect("worker request");
13249
13250        assert_eq!(
13251            server.authorization_for("/api/health").as_deref(),
13252            Some("Bearer shared-secret")
13253        );
13254        assert_eq!(
13255            server.control_protocol_for("/api/health").as_deref(),
13256            Some(CONTROL_PLANE_VERSION)
13257        );
13258        assert_eq!(
13259            server.authorization_for("/api/worker/register").as_deref(),
13260            Some("Bearer shared-secret")
13261        );
13262        assert_eq!(
13263            server
13264                .worker_protocol_for("/api/worker/register")
13265                .as_deref(),
13266            Some(WORKER_PROTOCOL_VERSION)
13267        );
13268    }
13269
13270    #[tokio::test]
13271    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
13272        let server = MockWorkerServer::start();
13273        let client = Client::builder(server.base_url())
13274            .timeout(Duration::from_secs(2))
13275            .build()
13276            .expect("client");
13277
13278        client
13279            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
13280            .await
13281            .expect("register");
13282        client
13283            .heartbeat_worker("capture-worker", 1, 1)
13284            .await
13285            .expect("heartbeat");
13286        client
13287            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13288            .await
13289            .expect("workflow poll");
13290        client
13291            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13292            .await
13293            .expect("activity poll");
13294
13295        for path in [
13296            "/api/worker/register",
13297            "/api/worker/heartbeat",
13298            "/api/worker/workflow-tasks/poll",
13299            "/api/worker/activity-tasks/poll",
13300        ] {
13301            assert_eq!(
13302                server.worker_protocol_for(path).as_deref(),
13303                Some(WORKER_PROTOCOL_VERSION),
13304                "unexpected protocol for {path}"
13305            );
13306        }
13307
13308        assert_eq!(
13309            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
13310            1
13311        );
13312        assert_eq!(
13313            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
13314            1
13315        );
13316        assert!(
13317            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
13318                .as_str()
13319                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
13320        );
13321        assert!(
13322            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
13323                .as_str()
13324                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
13325        );
13326    }
13327
13328    #[tokio::test]
13329    async fn query_task_endpoints_send_the_query_feature_protocol() {
13330        let server = MockWorkerServer::start();
13331        let client = Client::builder(server.base_url())
13332            .timeout(Duration::from_secs(2))
13333            .build()
13334            .expect("client");
13335
13336        client
13337            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13338            .await
13339            .expect("query poll");
13340        client
13341            .complete_query_task(
13342                "query-capture",
13343                "capture-worker",
13344                1,
13345                json!(8),
13346                DEFAULT_CODEC,
13347            )
13348            .await
13349            .expect("query complete");
13350        client
13351            .fail_query_task(
13352                "query-capture",
13353                "capture-worker",
13354                1,
13355                "failed",
13356                "query_rejected",
13357                "QueryFailed",
13358            )
13359            .await
13360            .expect("query fail");
13361
13362        for path in [
13363            "/api/worker/query-tasks/poll",
13364            "/api/worker/query-tasks/query-capture/complete",
13365            "/api/worker/query-tasks/query-capture/fail",
13366        ] {
13367            assert_eq!(
13368                server.worker_protocol_for(path).as_deref(),
13369                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
13370                "unexpected protocol for {path}"
13371            );
13372        }
13373
13374        assert_eq!(
13375            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
13376            1
13377        );
13378        assert!(
13379            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
13380                .as_str()
13381                .is_some_and(|id| id.starts_with("rust-query-poll-"))
13382        );
13383    }
13384
13385    #[tokio::test]
13386    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
13387        let server = MockWorkerServer::transient_worker_failures();
13388        let client = Client::builder(server.base_url())
13389            .timeout(Duration::from_secs(2))
13390            .build()
13391            .expect("client");
13392
13393        client
13394            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13395            .await
13396            .expect("workflow poll retry");
13397        client
13398            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13399            .await
13400            .expect("activity poll retry");
13401        client
13402            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13403            .await
13404            .expect("query poll retry");
13405
13406        for path in [
13407            "/api/worker/workflow-tasks/poll",
13408            "/api/worker/activity-tasks/poll",
13409            "/api/worker/query-tasks/poll",
13410        ] {
13411            let bodies = server.request_bodies(path);
13412            assert_eq!(bodies.len(), 2, "{path} must be retried once");
13413            assert_eq!(
13414                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
13415                "{path} must preserve the request binding across retry"
13416            );
13417        }
13418    }
13419
13420    #[tokio::test]
13421    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
13422        let server = MockWorkerServer::consecutive_poll_failures(2);
13423        let client = Client::builder(server.base_url())
13424            .timeout(Duration::from_secs(2))
13425            .build()
13426            .expect("client");
13427        let mut worker = Worker::new(client, "capture")
13428            .worker_id("capture-worker")
13429            .poll_timeout(Duration::from_millis(10))
13430            .retry_policy(WorkerRetryPolicy {
13431                max_retries: 2,
13432                initial_backoff: Duration::from_millis(1),
13433                max_backoff: Duration::from_millis(1),
13434            });
13435        worker.register_workflow(
13436            "capture.workflow",
13437            |_ctx, _input| async move { Ok(Value::Null) },
13438        );
13439        worker.register_activity(
13440            "capture.activity",
13441            |_ctx, _input| async move { Ok(Value::Null) },
13442        );
13443        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
13444            Ok(Value::Null)
13445        });
13446
13447        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
13448
13449        for path in [
13450            "/api/worker/workflow-tasks/poll",
13451            "/api/worker/activity-tasks/poll",
13452            "/api/worker/query-tasks/poll",
13453        ] {
13454            let bodies = server.request_bodies(path);
13455            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
13456            assert!(
13457                bodies
13458                    .iter()
13459                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
13460                "{path} must preserve one request binding across every retry"
13461            );
13462        }
13463    }
13464
13465    #[tokio::test]
13466    async fn query_protocol_rejection_from_older_server_is_typed() {
13467        let server = MockWorkerServer::reject_query_protocol();
13468        let client = Client::builder(server.base_url())
13469            .timeout(Duration::from_secs(2))
13470            .build()
13471            .expect("client");
13472
13473        let error = client
13474            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13475            .await
13476            .expect_err("server below query protocol floor must reject");
13477        let Error::Protocol(failure) = error else {
13478            panic!("expected typed protocol failure");
13479        };
13480
13481        assert_eq!(failure.status, 400);
13482        assert_eq!(failure.reason, "unsupported_protocol_version");
13483        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
13484        assert_eq!(
13485            failure.requested_version.as_deref(),
13486            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13487        );
13488        assert_eq!(
13489            server
13490                .worker_protocol_for("/api/worker/query-tasks/poll")
13491                .as_deref(),
13492            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13493        );
13494    }
13495
13496    #[tokio::test]
13497    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
13498        let server = MockWorkerServer::reject_query_protocol();
13499        let client = Client::builder(server.base_url())
13500            .timeout(Duration::from_secs(2))
13501            .build()
13502            .expect("client");
13503        let mut worker = Worker::new(client, "rust-workers")
13504            .worker_id("baseline-worker")
13505            .poll_timeout(Duration::from_millis(10));
13506
13507        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
13508            Ok(Value::Null)
13509        });
13510
13511        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
13512        assert_eq!(
13513            server
13514                .worker_protocol_for("/api/worker/workflow-tasks/poll")
13515                .as_deref(),
13516            Some(WORKER_PROTOCOL_VERSION)
13517        );
13518        assert_eq!(
13519            server.worker_protocol_for("/api/worker/query-tasks/poll"),
13520            None,
13521            "a worker without query handlers must not use the query-task endpoint"
13522        );
13523    }
13524
13525    #[tokio::test]
13526    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
13527        let server = MockWorkerServer::reject_query_completion();
13528        let client = Client::builder(server.base_url())
13529            .timeout(Duration::from_secs(2))
13530            .build()
13531            .expect("client");
13532
13533        let error = client
13534            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
13535            .await
13536            .expect_err("expired completion must be rejected");
13537        let Error::QueryFailed(failure) = error else {
13538            panic!("expected typed query failure");
13539        };
13540        assert_eq!(failure.status, 409);
13541        assert_eq!(failure.reason, "query_task_timed_out");
13542
13543        let mut worker = Worker::new(client, "rust-workers")
13544            .worker_id("late-worker")
13545            .poll_timeout(Duration::from_millis(10));
13546        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13547        worker.register_query(
13548            "counter",
13549            "current",
13550            |_ctx, _args| async move { Ok(json!(8)) },
13551        );
13552
13553        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
13554        assert_eq!(
13555            worker
13556                .run_once()
13557                .await
13558                .expect("worker continues after late completion"),
13559            0
13560        );
13561        assert_eq!(
13562            server.request_count("/api/worker/query-tasks/query-late/complete"),
13563            2
13564        );
13565        assert_eq!(
13566            server.request_count("/api/worker/query-tasks/query-late/fail"),
13567            0,
13568            "a server completion rejection must not be reported as an encoding failure"
13569        );
13570    }
13571
13572    #[tokio::test]
13573    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
13574        let server = MockWorkerServer::start();
13575        let client = Client::builder(server.base_url())
13576            .timeout(Duration::from_secs(2))
13577            .build()
13578            .expect("client");
13579        let mut worker = Worker::new(client, "rust-workers")
13580            .worker_id("joined-worker")
13581            .poll_timeout(Duration::from_millis(10));
13582        worker.register_workflow(
13583            "joined.workflow",
13584            |_ctx, _input| async move { Ok(Value::Null) },
13585        );
13586        worker.register_activity(
13587            "joined.activity",
13588            |_ctx, _input| async move { Ok(Value::Null) },
13589        );
13590        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
13591            Ok(Value::Null)
13592        });
13593
13594        worker
13595            .run_until(tokio::time::sleep(Duration::from_millis(20)))
13596            .await
13597            .expect("normal shutdown");
13598
13599        let deregistration_path = "/api/worker/registrations/mock-worker";
13600        assert_eq!(server.request_count(deregistration_path), 1);
13601        for poll_path in [
13602            "/api/worker/workflow-tasks/poll",
13603            "/api/worker/activity-tasks/poll",
13604            "/api/worker/query-tasks/poll",
13605        ] {
13606            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
13607        }
13608        assert_eq!(
13609            server.captured_paths().last().map(String::as_str),
13610            Some(deregistration_path),
13611            "deregistration must start only after every poller has joined"
13612        );
13613    }
13614
13615    #[tokio::test]
13616    async fn registration_failure_does_not_deregister() {
13617        let server = MockWorkerServer::rejected_registration();
13618        let client = Client::builder(server.base_url())
13619            .timeout(Duration::from_secs(2))
13620            .build()
13621            .expect("client");
13622        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
13623
13624        let error = worker
13625            .run_until(async {})
13626            .await
13627            .expect_err("registration must fail");
13628        assert!(matches!(
13629            error,
13630            Error::Http {
13631                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
13632                ..
13633            }
13634        ));
13635        assert!(server
13636            .captured_paths()
13637            .iter()
13638            .all(|path| !path.starts_with("/api/worker/registrations/")));
13639    }
13640
13641    #[tokio::test]
13642    async fn declined_registration_does_not_deregister() {
13643        let server = MockWorkerServer::declined_registration();
13644        let client = Client::builder(server.base_url())
13645            .timeout(Duration::from_secs(2))
13646            .build()
13647            .expect("client");
13648        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
13649
13650        let error = worker
13651            .run_until(async {})
13652            .await
13653            .expect_err("declined registration must fail");
13654        assert!(matches!(error, Error::WorkerLoop(_)));
13655        assert!(error.to_string().contains("was not accepted"));
13656        assert!(server
13657            .captured_paths()
13658            .iter()
13659            .all(|path| !path.starts_with("/api/worker/registrations/")));
13660    }
13661
13662    #[tokio::test]
13663    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
13664        let server = MockWorkerServer::rejected_deregistration();
13665        let client = Client::builder(server.base_url())
13666            .timeout(Duration::from_secs(2))
13667            .build()
13668            .expect("client");
13669        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
13670
13671        let error = worker
13672            .run_until(async {})
13673            .await
13674            .expect_err("deregistration must fail");
13675        assert!(matches!(
13676            error,
13677            Error::Http {
13678                status: reqwest::StatusCode::FORBIDDEN,
13679                ..
13680            }
13681        ));
13682        assert_eq!(
13683            server.request_count("/api/worker/registrations/mock-worker"),
13684            1
13685        );
13686    }
13687
13688    #[tokio::test]
13689    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13690        let server = MockWorkerServer::rejected_deregistration_protocol();
13691        let client = Client::builder(server.base_url())
13692            .timeout(Duration::from_secs(2))
13693            .build()
13694            .expect("client");
13695        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13696
13697        let error = worker
13698            .run_until(async {})
13699            .await
13700            .expect_err("protocol rejection must fail shutdown");
13701        let Error::Protocol(failure) = error else {
13702            panic!("expected typed protocol failure");
13703        };
13704        assert_eq!(failure.reason, "unsupported_protocol_version");
13705        assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13706        assert_eq!(
13707            server.request_count("/api/worker/registrations/mock-worker"),
13708            1
13709        );
13710    }
13711
13712    #[tokio::test]
13713    async fn primary_poller_error_retains_deregistration_failure_context() {
13714        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13715        let client = Client::builder(server.base_url())
13716            .timeout(Duration::from_secs(2))
13717            .build()
13718            .expect("client");
13719        let mut worker = Worker::new(client, "rust-workers")
13720            .worker_id("combined-failure")
13721            .poll_timeout(Duration::from_millis(10));
13722        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13723            Ok(Value::Null)
13724        });
13725
13726        let error = worker
13727            .run()
13728            .await
13729            .expect_err("worker and cleanup must fail");
13730        let summary = error.to_string();
13731        assert!(summary.contains("authentication_failed"));
13732        assert!(summary.contains("worker cannot deregister"));
13733        let Error::WorkerShutdown {
13734            primary,
13735            deregistration,
13736        } = error
13737        else {
13738            panic!("expected combined worker shutdown error");
13739        };
13740        assert!(matches!(
13741            *primary,
13742            Error::Http {
13743                status: reqwest::StatusCode::UNAUTHORIZED,
13744                ..
13745            }
13746        ));
13747        assert!(matches!(
13748            *deregistration,
13749            Error::Http {
13750                status: reqwest::StatusCode::FORBIDDEN,
13751                ..
13752            }
13753        ));
13754        assert_eq!(
13755            server.request_count("/api/worker/registrations/mock-worker"),
13756            1
13757        );
13758    }
13759
13760    #[tokio::test]
13761    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13762        let server = MockWorkerServer::start();
13763        let client = Client::builder(server.base_url())
13764            .timeout(Duration::from_secs(2))
13765            .build()
13766            .expect("client");
13767        let mut worker = Worker::new(client, "rust-workers")
13768            .worker_id("activity-only-worker")
13769            .poll_timeout(Duration::from_millis(10));
13770
13771        worker.register_activity(
13772            "activity.only",
13773            |_ctx, _args| async move { Ok(Value::Null) },
13774        );
13775
13776        worker.run_until(async {}).await.expect("run worker");
13777    }
13778
13779    #[tokio::test]
13780    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13781        let server = MockWorkerServer::start();
13782        let client = Client::builder(server.base_url())
13783            .timeout(Duration::from_secs(2))
13784            .build()
13785            .expect("client");
13786        let mut worker = Worker::new(client, "rust-workers")
13787            .worker_id("workflow-only-worker")
13788            .poll_timeout(Duration::from_millis(10));
13789
13790        worker.register_workflow(
13791            "workflow.only",
13792            |_ctx, _input| async move { Ok(Value::Null) },
13793        );
13794
13795        worker.run_until(async {}).await.expect("run worker");
13796    }
13797
13798    #[tokio::test]
13799    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13800        let server = MockWorkerServer::start();
13801        let client = Client::builder(server.base_url())
13802            .timeout(Duration::from_secs(2))
13803            .build()
13804            .expect("client");
13805        let observations = Arc::new(Mutex::new(Vec::new()));
13806        let observed = Arc::clone(&observations);
13807        let mut worker = Worker::new(client, "rust-workers")
13808            .worker_id("observed-heartbeat-worker")
13809            .poll_timeout(Duration::from_millis(10))
13810            .on_worker_heartbeat(move |observation| {
13811                observed
13812                    .lock()
13813                    .expect("heartbeat observations")
13814                    .push(observation.clone());
13815            });
13816
13817        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13818            Ok(Value::Null)
13819        });
13820        let acknowledged = Arc::clone(&observations);
13821        worker
13822            .run_until(async move {
13823                tokio::time::timeout(Duration::from_secs(2), async move {
13824                    loop {
13825                        if !acknowledged
13826                            .lock()
13827                            .expect("heartbeat observations")
13828                            .is_empty()
13829                        {
13830                            break;
13831                        }
13832                        tokio::time::sleep(Duration::from_millis(1)).await;
13833                    }
13834                })
13835                .await
13836                .expect("heartbeat acknowledgement within timeout");
13837            })
13838            .await
13839            .expect("run worker");
13840
13841        let observations = observations.lock().expect("heartbeat observations");
13842        let first = observations.first().expect("heartbeat acknowledgement");
13843        assert_eq!(first.worker_id, "observed-heartbeat-worker");
13844        assert_eq!(first.task_queue, "rust-workers");
13845        assert!(first.acknowledged_at_unix_millis > 0);
13846        assert_eq!(first.acknowledgement, json!({}));
13847    }
13848
13849    #[tokio::test]
13850    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13851        let server = MockWorkerServer::delayed_heartbeat_worker();
13852        let client = Client::builder(server.base_url())
13853            .timeout(Duration::from_secs(3))
13854            .build()
13855            .expect("client");
13856        let observations = Arc::new(Mutex::new(Vec::new()));
13857        let observed = Arc::clone(&observations);
13858        let mut worker = Worker::new(client, "rust-snapshot-workers")
13859            .worker_id("rust-snapshot-worker")
13860            .poll_timeout(Duration::from_millis(10))
13861            .on_worker_heartbeat(move |observation| {
13862                observed
13863                    .lock()
13864                    .expect("heartbeat observations")
13865                    .push(observation.clone());
13866            });
13867
13868        worker.register_workflow("snapshot", |ctx, _input| async move {
13869            ctx.wait_signal("finish").await?;
13870            Ok(json!({"status": "finished"}))
13871        });
13872        worker.register_query("snapshot", "current", |ctx, _args| async move {
13873            Ok(json!(ctx
13874                .signals("increment")
13875                .iter()
13876                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13877                .sum::<i64>()))
13878        });
13879        worker.register_activity("cancel-aware", |_ctx, _args| async move {
13880            Ok(json!({"late": "completion"}))
13881        });
13882
13883        worker
13884            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13885            .await
13886            .expect("delayed heartbeat must allow a clean worker shutdown");
13887
13888        let observations = observations.lock().expect("heartbeat observations");
13889        assert!(
13890            observations.len() >= 3,
13891            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13892        );
13893        assert!(
13894            observations.windows(2).all(|pair| {
13895                pair[1].acknowledged_at_unix_millis
13896                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13897                    >= 850
13898            }),
13899            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13900        );
13901        drop(observations);
13902
13903        let heartbeat_times = server.request_times("/api/worker/heartbeat");
13904        let delayed_request_at = *heartbeat_times
13905            .get(1)
13906            .expect("intentionally delayed heartbeat request");
13907        let delay_window_start = delayed_request_at + Duration::from_millis(100);
13908        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13909        for path in [
13910            "/api/worker/workflow-tasks/poll",
13911            "/api/worker/activity-tasks/poll",
13912            "/api/worker/query-tasks/poll",
13913        ] {
13914            assert!(
13915                server
13916                    .request_times(path)
13917                    .iter()
13918                    .any(|received_at| *received_at >= delay_window_start
13919                        && *received_at <= delay_window_end),
13920                "{path} must keep polling while a heartbeat acknowledgement is delayed"
13921            );
13922        }
13923        assert!(
13924            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13925            "workflow work must be settled"
13926        );
13927        assert!(
13928            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13929            "activity work must be settled"
13930        );
13931        assert!(
13932            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13933            "query work must be settled"
13934        );
13935    }
13936
13937    #[tokio::test]
13938    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13939        let server = MockWorkerServer::heartbeat_retry_worker();
13940        let client = Client::builder(server.base_url())
13941            .timeout(Duration::from_secs(2))
13942            .build()
13943            .expect("client");
13944        let observations = Arc::new(Mutex::new(Vec::new()));
13945        let observed = Arc::clone(&observations);
13946        let worker = Worker::new(client, "rust-workers")
13947            .worker_id("heartbeat-retry-worker")
13948            .retry_policy(WorkerRetryPolicy {
13949                max_retries: 1,
13950                initial_backoff: Duration::from_millis(300),
13951                max_backoff: Duration::from_millis(300),
13952            })
13953            .on_worker_heartbeat(move |observation| {
13954                observed
13955                    .lock()
13956                    .expect("heartbeat observations")
13957                    .push(observation.clone());
13958            });
13959
13960        worker
13961            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13962            .await
13963            .expect("retryable heartbeat failure must remain bounded and recover");
13964
13965        let observations = observations.lock().expect("heartbeat observations");
13966        assert!(observations.len() >= 3, "heartbeat retry must recover");
13967        assert!(
13968            observations.windows(2).all(|pair| {
13969                pair[1]
13970                    .acknowledged_at_unix_millis
13971                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13972                    >= 850
13973            }),
13974            "a successful retry must start a fresh advertised cadence: {observations:?}"
13975        );
13976        assert_eq!(
13977            server.request_count("/api/worker/heartbeat"),
13978            observations.len() + 1,
13979            "one retryable failure must add exactly one bounded request"
13980        );
13981    }
13982
13983    #[tokio::test]
13984    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13985        let server = MockWorkerServer::waiting_query_worker();
13986        let client = Client::builder(server.base_url())
13987            .timeout(Duration::from_secs(2))
13988            .build()
13989            .expect("client");
13990        let observations = Arc::new(Mutex::new(Vec::new()));
13991        let observed = Arc::clone(&observations);
13992        let mut worker = Worker::new(client, "rust-snapshot-workers")
13993            .worker_id("rust-snapshot-worker")
13994            .poll_timeout(Duration::from_millis(10))
13995            .on_worker_heartbeat(move |observation| {
13996                observed
13997                    .lock()
13998                    .expect("heartbeat observations")
13999                    .push(observation.clone());
14000            });
14001
14002        worker.register_workflow("snapshot", |ctx, _input| async move {
14003            ctx.wait_signal("finish").await?;
14004            Ok(json!({"status": "finished"}))
14005        });
14006        worker.register_query("snapshot", "current", |ctx, _args| async move {
14007            let current = ctx
14008                .signals("increment")
14009                .iter()
14010                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
14011                .sum::<i64>();
14012            Ok(json!(current))
14013        });
14014        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
14015
14016        worker
14017            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
14018            .await
14019            .expect("pending workflow and query poller must remain live until shutdown");
14020
14021        assert!(
14022            observations.lock().expect("heartbeat observations").len() >= 4,
14023            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
14024        );
14025        assert!(
14026            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
14027            "workflow polling must continue after empty replay acknowledgements"
14028        );
14029        assert!(
14030            server.request_count("/api/worker/query-tasks/poll") >= 2,
14031            "query polling must continue after serving the current query"
14032        );
14033        assert_eq!(
14034            server.request_body("/api/worker/register")["capabilities"],
14035            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
14036        );
14037        assert_eq!(
14038            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
14039            json!({
14040                "queries": ["current"],
14041                "updates": ["replace"],
14042                "update_validators": [],
14043            })
14044        );
14045
14046        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
14047        assert_eq!(
14048            opened["commands"],
14049            json!([{
14050                "type": "open_signal_wait",
14051                "signal_name": "finish",
14052            }])
14053        );
14054
14055        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
14056            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
14057            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
14058            let failure = server.request_body(&fail_path);
14059            assert_eq!(
14060                failure["failure"]["type"],
14061                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
14062            );
14063            assert_eq!(server.request_count(&completion_path), 0);
14064        }
14065
14066        let query_completion =
14067            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
14068        assert_eq!(query_completion["result"], json!(8));
14069
14070        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
14071        assert_eq!(
14072            server.request_count(terminal_path),
14073            1,
14074            "the matching signal must settle the workflow exactly once"
14075        );
14076        let terminal = server.request_body(terminal_path);
14077        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
14078        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
14079        assert_eq!(
14080            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
14081                .expect("terminal workflow result"),
14082            json!({"status": "finished"})
14083        );
14084    }
14085
14086    #[tokio::test]
14087    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
14088        let server = MockWorkerServer::transient_worker_failures();
14089        let client = Client::builder(server.base_url())
14090            .timeout(Duration::from_secs(2))
14091            .build()
14092            .expect("client");
14093        let mut worker = Worker::new(client, "rust-workers")
14094            .worker_id("retry-worker")
14095            .poll_timeout(Duration::from_millis(10))
14096            .retry_policy(WorkerRetryPolicy {
14097                max_retries: 2,
14098                initial_backoff: Duration::from_millis(1),
14099                max_backoff: Duration::from_millis(1),
14100            });
14101        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14102        worker.register_activity(
14103            "counter.activity",
14104            |_ctx, _input| async move { Ok(Value::Null) },
14105        );
14106        worker.register_query(
14107            "counter",
14108            "current",
14109            |_ctx, _args| async move { Ok(json!(8)) },
14110        );
14111
14112        worker
14113            .run_until(tokio::time::sleep(Duration::from_millis(75)))
14114            .await
14115            .expect("transient failures must not stop the worker");
14116
14117        for path in [
14118            "/api/worker/heartbeat",
14119            "/api/worker/workflow-tasks/poll",
14120            "/api/worker/activity-tasks/poll",
14121            "/api/worker/query-tasks/poll",
14122        ] {
14123            assert!(
14124                server.request_count(path) >= 2,
14125                "{path} must continue after its transient failure"
14126            );
14127        }
14128    }
14129
14130    #[tokio::test]
14131    async fn worker_bounds_transport_retries() {
14132        let server = MockWorkerServer::unavailable_polls();
14133        let client = Client::builder(server.base_url())
14134            .timeout(Duration::from_secs(2))
14135            .build()
14136            .expect("client");
14137        let mut worker = Worker::new(client, "rust-workers")
14138            .worker_id("bounded-retry-worker")
14139            .poll_timeout(Duration::from_millis(10))
14140            .retry_policy(WorkerRetryPolicy {
14141                max_retries: 2,
14142                initial_backoff: Duration::from_millis(1),
14143                max_backoff: Duration::from_millis(1),
14144            });
14145        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14146
14147        let error = worker.run().await.expect_err("retry bound must terminate");
14148        assert!(matches!(error, Error::Transport(_)));
14149        assert_eq!(
14150            server.request_count("/api/worker/workflow-tasks/poll"),
14151            3,
14152            "one initial request plus exactly two retries"
14153        );
14154    }
14155
14156    #[tokio::test]
14157    async fn worker_retry_policy_can_disable_poll_retries() {
14158        let server = MockWorkerServer::unavailable_polls();
14159        let client = Client::builder(server.base_url())
14160            .timeout(Duration::from_secs(2))
14161            .build()
14162            .expect("client");
14163        let mut worker = Worker::new(client, "rust-workers")
14164            .worker_id("no-retry-worker")
14165            .poll_timeout(Duration::from_millis(10))
14166            .retry_policy(WorkerRetryPolicy {
14167                max_retries: 0,
14168                initial_backoff: Duration::from_millis(1),
14169                max_backoff: Duration::from_millis(1),
14170            });
14171        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14172
14173        let error = worker
14174            .run_once()
14175            .await
14176            .expect_err("disabled retries must return the first transport failure");
14177        assert!(matches!(error, Error::Transport(_)));
14178        assert_eq!(
14179            server.request_count("/api/worker/workflow-tasks/poll"),
14180            1,
14181            "max_retries=0 must send only the initial request"
14182        );
14183    }
14184
14185    #[tokio::test]
14186    async fn worker_does_not_retry_authentication_failures() {
14187        let server = MockWorkerServer::unauthorized_polls();
14188        let client = Client::builder(server.base_url())
14189            .timeout(Duration::from_secs(2))
14190            .build()
14191            .expect("client");
14192        let mut worker = Worker::new(client, "rust-workers")
14193            .worker_id("unauthorized-worker")
14194            .poll_timeout(Duration::from_millis(10));
14195        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14196
14197        let error = worker
14198            .run()
14199            .await
14200            .expect_err("authentication must terminate");
14201        let Error::Http { status, body } = error else {
14202            panic!("expected stable HTTP authentication error");
14203        };
14204        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
14205        assert!(body.contains("authentication_failed"));
14206        assert_eq!(
14207            server.request_count("/api/worker/workflow-tasks/poll"),
14208            1,
14209            "authentication failures must not be retried"
14210        );
14211    }
14212
14213    #[derive(Clone, Debug)]
14214    struct CapturedRequest {
14215        method: String,
14216        path: String,
14217        authorization: Option<String>,
14218        namespace: Option<String>,
14219        worker_protocol: Option<String>,
14220        control_protocol: Option<String>,
14221        body: String,
14222        received_at: Instant,
14223    }
14224
14225    struct MockWorkerServer {
14226        addr: SocketAddr,
14227        stop: Arc<AtomicBool>,
14228        requests: Arc<Mutex<Vec<CapturedRequest>>>,
14229        thread: Option<thread::JoinHandle<()>>,
14230    }
14231
14232    #[derive(Clone, Copy, Default)]
14233    struct MockWorkerBehavior {
14234        reject_query_protocol: bool,
14235        reject_query_completion: bool,
14236        waiting_query_worker: bool,
14237        decline_registration: bool,
14238        complete_named_signal: bool,
14239        poll_failures_per_path: usize,
14240        heartbeat_failures: usize,
14241        heartbeat_failure_request: Option<usize>,
14242        delayed_heartbeat_request: Option<usize>,
14243        heartbeat_response_delay: Duration,
14244        concurrent_requests: bool,
14245        unauthorized_polls: bool,
14246        reject_registration: bool,
14247        reject_deregistration: bool,
14248        reject_deregistration_protocol: bool,
14249        cancelled_activity: bool,
14250        draining_polls: bool,
14251        workflow_completion_status: Option<&'static str>,
14252        workflow_completion_body: Option<&'static str>,
14253    }
14254
14255    impl MockWorkerServer {
14256        fn start() -> Self {
14257            Self::start_with_behavior(MockWorkerBehavior::default())
14258        }
14259
14260        fn reject_query_protocol() -> Self {
14261            Self::start_with_behavior(MockWorkerBehavior {
14262                reject_query_protocol: true,
14263                ..MockWorkerBehavior::default()
14264            })
14265        }
14266
14267        fn reject_query_completion() -> Self {
14268            Self::start_with_behavior(MockWorkerBehavior {
14269                reject_query_completion: true,
14270                ..MockWorkerBehavior::default()
14271            })
14272        }
14273
14274        fn waiting_query_worker() -> Self {
14275            Self::start_with_behavior(MockWorkerBehavior {
14276                waiting_query_worker: true,
14277                complete_named_signal: true,
14278                ..MockWorkerBehavior::default()
14279            })
14280        }
14281
14282        fn transient_worker_failures() -> Self {
14283            Self::start_with_behavior(MockWorkerBehavior {
14284                poll_failures_per_path: 1,
14285                heartbeat_failures: 1,
14286                ..MockWorkerBehavior::default()
14287            })
14288        }
14289
14290        fn consecutive_poll_failures(count: usize) -> Self {
14291            Self::start_with_behavior(MockWorkerBehavior {
14292                poll_failures_per_path: count,
14293                ..MockWorkerBehavior::default()
14294            })
14295        }
14296
14297        fn delayed_heartbeat_worker() -> Self {
14298            Self::start_with_behavior(MockWorkerBehavior {
14299                waiting_query_worker: true,
14300                delayed_heartbeat_request: Some(2),
14301                heartbeat_response_delay: Duration::from_millis(1_500),
14302                concurrent_requests: true,
14303                cancelled_activity: true,
14304                ..MockWorkerBehavior::default()
14305            })
14306        }
14307
14308        fn heartbeat_retry_worker() -> Self {
14309            Self::start_with_behavior(MockWorkerBehavior {
14310                waiting_query_worker: true,
14311                heartbeat_failure_request: Some(2),
14312                concurrent_requests: true,
14313                ..MockWorkerBehavior::default()
14314            })
14315        }
14316
14317        fn unavailable_polls() -> Self {
14318            Self::start_with_behavior(MockWorkerBehavior {
14319                poll_failures_per_path: usize::MAX,
14320                ..MockWorkerBehavior::default()
14321            })
14322        }
14323
14324        fn unauthorized_polls() -> Self {
14325            Self::start_with_behavior(MockWorkerBehavior {
14326                unauthorized_polls: true,
14327                ..MockWorkerBehavior::default()
14328            })
14329        }
14330
14331        fn rejected_registration() -> Self {
14332            Self::start_with_behavior(MockWorkerBehavior {
14333                reject_registration: true,
14334                ..MockWorkerBehavior::default()
14335            })
14336        }
14337
14338        fn declined_registration() -> Self {
14339            Self::start_with_behavior(MockWorkerBehavior {
14340                decline_registration: true,
14341                ..MockWorkerBehavior::default()
14342            })
14343        }
14344
14345        fn rejected_deregistration() -> Self {
14346            Self::start_with_behavior(MockWorkerBehavior {
14347                reject_deregistration: true,
14348                ..MockWorkerBehavior::default()
14349            })
14350        }
14351
14352        fn rejected_deregistration_protocol() -> Self {
14353            Self::start_with_behavior(MockWorkerBehavior {
14354                reject_deregistration_protocol: true,
14355                ..MockWorkerBehavior::default()
14356            })
14357        }
14358
14359        fn unauthorized_polls_and_rejected_deregistration() -> Self {
14360            Self::start_with_behavior(MockWorkerBehavior {
14361                unauthorized_polls: true,
14362                reject_deregistration: true,
14363                ..MockWorkerBehavior::default()
14364            })
14365        }
14366
14367        fn cancelled_activity() -> Self {
14368            Self::start_with_behavior(MockWorkerBehavior {
14369                cancelled_activity: true,
14370                ..MockWorkerBehavior::default()
14371            })
14372        }
14373
14374        fn draining_polls() -> Self {
14375            Self::start_with_behavior(MockWorkerBehavior {
14376                draining_polls: true,
14377                ..MockWorkerBehavior::default()
14378            })
14379        }
14380
14381        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
14382            Self::start_with_behavior(MockWorkerBehavior {
14383                workflow_completion_status: Some(status),
14384                workflow_completion_body: Some(body),
14385                ..MockWorkerBehavior::default()
14386            })
14387        }
14388
14389        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
14390            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
14391            listener
14392                .set_nonblocking(true)
14393                .expect("configure mock listener");
14394            let addr = listener.local_addr().expect("mock server address");
14395            let stop = Arc::new(AtomicBool::new(false));
14396            let server_stop = Arc::clone(&stop);
14397            let requests = Arc::new(Mutex::new(Vec::new()));
14398            let server_requests = Arc::clone(&requests);
14399            let thread = thread::spawn(move || {
14400                let mut request_threads = Vec::new();
14401                while !server_stop.load(Ordering::SeqCst) {
14402                    match listener.accept() {
14403                        Ok((mut stream, _)) => {
14404                            if behavior.concurrent_requests {
14405                                let requests = Arc::clone(&server_requests);
14406                                request_threads.push(thread::spawn(move || {
14407                                    handle_mock_worker_request(&mut stream, &requests, behavior)
14408                                }));
14409                            } else {
14410                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
14411                            }
14412                        }
14413                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
14414                            let mut index = 0;
14415                            while index < request_threads.len() {
14416                                if request_threads[index].is_finished() {
14417                                    request_threads
14418                                        .swap_remove(index)
14419                                        .join()
14420                                        .expect("join mock request");
14421                                } else {
14422                                    index += 1;
14423                                }
14424                            }
14425                            thread::sleep(Duration::from_millis(5));
14426                        }
14427                        Err(_) => break,
14428                    }
14429                }
14430                for request_thread in request_threads {
14431                    request_thread.join().expect("join mock request");
14432                }
14433            });
14434
14435            Self {
14436                addr,
14437                stop,
14438                requests,
14439                thread: Some(thread),
14440            }
14441        }
14442
14443        fn base_url(&self) -> String {
14444            format!("http://{}", self.addr)
14445        }
14446
14447        fn worker_protocol_for(&self, path: &str) -> Option<String> {
14448            self.requests
14449                .lock()
14450                .expect("captured requests")
14451                .iter()
14452                .find(|request| request.path == path)
14453                .and_then(|request| request.worker_protocol.clone())
14454        }
14455
14456        fn control_protocol_for(&self, path: &str) -> Option<String> {
14457            self.requests
14458                .lock()
14459                .expect("captured requests")
14460                .iter()
14461                .find(|request| request.path == path)
14462                .and_then(|request| request.control_protocol.clone())
14463        }
14464
14465        fn method_for(&self, path: &str) -> Option<String> {
14466            self.requests
14467                .lock()
14468                .expect("captured requests")
14469                .iter()
14470                .find(|request| request.path == path)
14471                .map(|request| request.method.clone())
14472        }
14473
14474        fn authorization_for(&self, path: &str) -> Option<String> {
14475            self.requests
14476                .lock()
14477                .expect("captured requests")
14478                .iter()
14479                .find(|request| request.path == path)
14480                .and_then(|request| request.authorization.clone())
14481        }
14482
14483        fn namespace_for(&self, path: &str) -> Option<String> {
14484            self.requests
14485                .lock()
14486                .expect("captured requests")
14487                .iter()
14488                .find(|request| request.path == path)
14489                .and_then(|request| request.namespace.clone())
14490        }
14491
14492        fn request_count(&self, path: &str) -> usize {
14493            self.requests
14494                .lock()
14495                .expect("captured requests")
14496                .iter()
14497                .filter(|request| request.path == path)
14498                .count()
14499        }
14500
14501        fn captured_paths(&self) -> Vec<String> {
14502            self.requests
14503                .lock()
14504                .expect("captured requests")
14505                .iter()
14506                .map(|request| request.path.clone())
14507                .collect()
14508        }
14509
14510        fn request_times(&self, path: &str) -> Vec<Instant> {
14511            self.requests
14512                .lock()
14513                .expect("captured requests")
14514                .iter()
14515                .filter(|request| request.path == path)
14516                .map(|request| request.received_at)
14517                .collect()
14518        }
14519
14520        fn request_body(&self, path: &str) -> Value {
14521            let requests = self.requests.lock().expect("captured requests");
14522            let body = &requests
14523                .iter()
14524                .find(|request| request.path == path)
14525                .unwrap_or_else(|| panic!("missing request for {path}"))
14526                .body;
14527            serde_json::from_str(body).unwrap_or_else(|error| {
14528                panic!("invalid JSON request body for {path}: {error}: {body:?}")
14529            })
14530        }
14531
14532        fn request_bodies(&self, path: &str) -> Vec<Value> {
14533            self.requests
14534                .lock()
14535                .expect("captured requests")
14536                .iter()
14537                .filter(|request| request.path == path)
14538                .map(|request| {
14539                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
14540                        panic!(
14541                            "invalid JSON request body for {path}: {error}: {:?}",
14542                            request.body
14543                        )
14544                    })
14545                })
14546                .collect()
14547        }
14548    }
14549
14550    impl Drop for MockWorkerServer {
14551        fn drop(&mut self) {
14552            self.stop.store(true, Ordering::SeqCst);
14553            let _ = TcpStream::connect(self.addr);
14554
14555            if let Some(thread) = self.thread.take() {
14556                thread.join().expect("join mock server");
14557            }
14558        }
14559    }
14560
14561    fn handle_mock_worker_request(
14562        stream: &mut TcpStream,
14563        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
14564        behavior: MockWorkerBehavior,
14565    ) {
14566        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
14567        let mut buffer = [0_u8; 8192];
14568        let mut request = Vec::new();
14569
14570        loop {
14571            match stream.read(&mut buffer) {
14572                Ok(0) => break,
14573                Ok(read) => {
14574                    request.extend_from_slice(&buffer[..read]);
14575                    if mock_request_is_complete(&request) {
14576                        break;
14577                    }
14578                }
14579                Err(error)
14580                    if matches!(
14581                        error.kind(),
14582                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
14583                    ) =>
14584                {
14585                    break;
14586                }
14587                Err(_) => return,
14588            }
14589        }
14590
14591        let request = String::from_utf8_lossy(&request);
14592        let body = request
14593            .split_once("\r\n\r\n")
14594            .map(|(_, body)| body)
14595            .unwrap_or_default();
14596        let path = request
14597            .lines()
14598            .next()
14599            .and_then(|line| line.split_whitespace().nth(1))
14600            .unwrap_or_default();
14601        let method = request
14602            .lines()
14603            .next()
14604            .and_then(|line| line.split_whitespace().next())
14605            .unwrap_or_default();
14606        let authorization = request.lines().find_map(|line| {
14607            let (name, value) = line.split_once(':')?;
14608            name.eq_ignore_ascii_case("Authorization")
14609                .then(|| value.trim().to_string())
14610        });
14611        let namespace = request.lines().find_map(|line| {
14612            let (name, value) = line.split_once(':')?;
14613            name.eq_ignore_ascii_case("X-Namespace")
14614                .then(|| value.trim().to_string())
14615        });
14616        let worker_protocol = request.lines().find_map(|line| {
14617            let (name, value) = line.split_once(':')?;
14618            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
14619                .then(|| value.trim().to_string())
14620        });
14621        let control_protocol = request.lines().find_map(|line| {
14622            let (name, value) = line.split_once(':')?;
14623            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
14624                .then(|| value.trim().to_string())
14625        });
14626        let request_number = {
14627            let mut requests = requests.lock().expect("captured requests");
14628            requests.push(CapturedRequest {
14629                method: method.to_string(),
14630                path: path.to_string(),
14631                authorization,
14632                namespace,
14633                worker_protocol: worker_protocol.clone(),
14634                control_protocol,
14635                body: body.to_string(),
14636                received_at: Instant::now(),
14637            });
14638            requests
14639                .iter()
14640                .filter(|request| request.path == path)
14641                .count()
14642        };
14643
14644        if behavior.reject_registration && path == "/api/worker/register" {
14645            write_mock_response(
14646                stream,
14647                "503 Service Unavailable",
14648                r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
14649            );
14650            return;
14651        }
14652
14653        if path.starts_with("/api/worker/registrations/") {
14654            if behavior.reject_deregistration_protocol {
14655                write_mock_response(
14656                    stream,
14657                    "400 Bad Request",
14658                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
14659                );
14660            } else if behavior.reject_deregistration {
14661                write_mock_response(
14662                    stream,
14663                    "403 Forbidden",
14664                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
14665                );
14666            } else {
14667                write_mock_response(
14668                    stream,
14669                    "200 OK",
14670                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
14671                );
14672            }
14673            return;
14674        }
14675
14676        let is_poll = matches!(
14677            path,
14678            "/api/worker/workflow-tasks/poll"
14679                | "/api/worker/activity-tasks/poll"
14680                | "/api/worker/query-tasks/poll"
14681        );
14682        if is_poll && request_number <= behavior.poll_failures_per_path {
14683            return;
14684        }
14685        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14686            return;
14687        }
14688        if path == "/api/worker/heartbeat"
14689            && behavior.heartbeat_failure_request == Some(request_number)
14690        {
14691            return;
14692        }
14693        if path == "/api/worker/heartbeat"
14694            && behavior.delayed_heartbeat_request == Some(request_number)
14695        {
14696            thread::sleep(behavior.heartbeat_response_delay);
14697        }
14698        if behavior.unauthorized_polls && is_poll {
14699            write_mock_response(
14700                stream,
14701                "401 Unauthorized",
14702                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14703            );
14704            return;
14705        }
14706        if behavior.draining_polls && is_poll {
14707            write_mock_response(
14708                stream,
14709                "409 Conflict",
14710                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14711            );
14712            return;
14713        }
14714
14715        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14716            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14717            let body = format!(
14718                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14719            );
14720            write_mock_response(stream, "400 Bad Request", &body);
14721            return;
14722        }
14723
14724        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14725        {
14726            write_mock_response(
14727                stream,
14728                "409 Conflict",
14729                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14730            );
14731            return;
14732        }
14733
14734        if behavior.workflow_completion_status.is_some()
14735            && path == "/api/worker/workflow-tasks/poll"
14736            && request_number == 1
14737        {
14738            write_mock_response(
14739                stream,
14740                "200 OK",
14741                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14742            );
14743            return;
14744        }
14745
14746        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14747            if let (Some(status), Some(body)) = (
14748                behavior.workflow_completion_status,
14749                behavior.workflow_completion_body,
14750            ) {
14751                write_mock_response(stream, status, body);
14752                return;
14753            }
14754        }
14755
14756        if behavior.waiting_query_worker {
14757            if behavior.complete_named_signal
14758                && path == "/api/worker/workflow-tasks/poll"
14759                && request_number == 1
14760            {
14761                let body = json!({
14762                    "task": {
14763                        "task_id": "snapshot-open",
14764                        "workflow_id": "snapshot-1",
14765                        "run_id": "snapshot-run-1",
14766                        "workflow_type": "snapshot",
14767                        "payload_codec": DEFAULT_CODEC,
14768                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14769                            .expect("Avro workflow arguments"),
14770                        "history_events": [],
14771                        "workflow_task_attempt": 1,
14772                        "lease_owner": "rust-snapshot-worker"
14773                    }
14774                })
14775                .to_string();
14776                write_mock_response(stream, "200 OK", &body);
14777                return;
14778            }
14779
14780            let signal_request = request_number - usize::from(behavior.complete_named_signal);
14781            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14782            if path == "/api/worker/workflow-tasks/poll"
14783                && signal_request >= 1
14784                && signal_request <= signal_request_limit
14785            {
14786                let finish = behavior.complete_named_signal && signal_request == 3;
14787                let amounts = if signal_request == 1 {
14788                    vec![3]
14789                } else {
14790                    vec![3, 5]
14791                };
14792                let task_id = if signal_request == 1 {
14793                    "snapshot-wait-3"
14794                } else if finish {
14795                    "snapshot-finish"
14796                } else {
14797                    "snapshot-wait-5"
14798                };
14799                let mut history_events = std::iter::once(json!({
14800                    "event_type": "SignalWaitOpened",
14801                    "payload": {"sequence": 1, "signal_name": "finish"}
14802                }))
14803                .chain(amounts.iter().enumerate().map(|(index, amount)| {
14804                    json!({
14805                        "event_type": "SignalReceived",
14806                        "payload": {
14807                            "signal_id": format!("increment-{amount}"),
14808                            "signal_name": "increment",
14809                            "workflow_sequence": index + 2,
14810                            "payload_codec": DEFAULT_CODEC,
14811                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14812                                .expect("Avro signal envelope")
14813                        }
14814                    })
14815                }))
14816                .collect::<Vec<_>>();
14817                let (resume_id, resume_name, resume_arguments) = if finish {
14818                    history_events.push(json!({
14819                        "event_type": "SignalReceived",
14820                        "payload": {
14821                            "signal_id": "finish",
14822                            "signal_name": "finish",
14823                            "workflow_sequence": 4,
14824                            "payload_codec": DEFAULT_CODEC,
14825                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14826                                .expect("Avro finish signal envelope")
14827                        }
14828                    }));
14829                    (
14830                        "finish".to_string(),
14831                        "finish".to_string(),
14832                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
14833                            .expect("Avro finish resume signal"),
14834                    )
14835                } else {
14836                    let amount = amounts.last().expect("amount");
14837                    (
14838                        format!("increment-{amount}"),
14839                        "increment".to_string(),
14840                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14841                            .expect("Avro increment resume signal"),
14842                    )
14843                };
14844                let body = json!({
14845                    "task": {
14846                        "task_id": task_id,
14847                        "workflow_id": "snapshot-1",
14848                        "run_id": "snapshot-run-1",
14849                        "workflow_type": "snapshot",
14850                        "payload_codec": DEFAULT_CODEC,
14851                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14852                            .expect("Avro workflow arguments"),
14853                        "history_events": history_events,
14854                        "workflow_task_attempt": 1,
14855                        "workflow_signal_id": resume_id,
14856                        "signal_name": resume_name,
14857                        "signal_arguments": resume_arguments,
14858                        "lease_owner": "rust-snapshot-worker"
14859                    }
14860                })
14861                .to_string();
14862                write_mock_response(stream, "200 OK", &body);
14863                return;
14864            }
14865
14866            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14867                let history_events = [3, 5]
14868                    .into_iter()
14869                    .enumerate()
14870                    .map(|(index, amount)| {
14871                        json!({
14872                            "event_type": "SignalReceived",
14873                            "payload": {
14874                                "signal_id": format!("increment-{amount}"),
14875                                "signal_name": "increment",
14876                                "workflow_sequence": index + 2,
14877                                "payload_codec": DEFAULT_CODEC,
14878                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14879                                    .expect("Avro query signal envelope")
14880                            }
14881                        })
14882                    })
14883                    .collect::<Vec<_>>();
14884                let body = json!({
14885                    "task": {
14886                        "query_task_id": "snapshot-current",
14887                        "query_task_attempt": 1,
14888                        "lease_owner": "rust-snapshot-worker",
14889                        "workflow_id": "snapshot-1",
14890                        "run_id": "snapshot-run-1",
14891                        "workflow_type": "snapshot",
14892                        "query_name": "current",
14893                        "payload_codec": DEFAULT_CODEC,
14894                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14895                            .expect("Avro workflow arguments"),
14896                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14897                            .expect("Avro query arguments"),
14898                        "history_events": history_events,
14899                        "run_status": "waiting"
14900                    }
14901                })
14902                .to_string();
14903                write_mock_response(stream, "200 OK", &body);
14904                return;
14905            }
14906
14907            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14908                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14909            {
14910                write_mock_response(
14911                    stream,
14912                    "200 OK",
14913                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
14914                );
14915                return;
14916            }
14917
14918            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14919                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14920                return;
14921            }
14922
14923            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14924                write_mock_response(
14925                    stream,
14926                    "200 OK",
14927                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14928                );
14929                return;
14930            }
14931
14932            if path == "/api/worker/query-tasks/snapshot-current/complete" {
14933                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14934                return;
14935            }
14936        }
14937
14938        if matches!(
14939            path,
14940            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14941        ) {
14942            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14943                .expect("typed mock result");
14944            let body = json!({
14945                "result": typed_fidelity_probe().into_json().expect("result projection"),
14946                "result_envelope": result,
14947            })
14948            .to_string();
14949            write_mock_response(stream, "200 OK", &body);
14950            return;
14951        }
14952
14953        if path == "/api/workflows/typed-1" {
14954            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14955                .expect("typed mock result");
14956            let body = json!({
14957                "workflow_id": "typed-1",
14958                "run_id": "run-typed-1",
14959                "workflow_type": "typed.echo",
14960                "status": "completed",
14961                "output": typed_fidelity_probe().into_json().expect("output projection"),
14962                "output_envelope": result,
14963            })
14964            .to_string();
14965            write_mock_response(stream, "200 OK", &body);
14966            return;
14967        }
14968
14969        let (status, body) = match path {
14970            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14971            "/api/workflows" => (
14972                "201 Created",
14973                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14974            ),
14975            "/api/worker/register" if behavior.decline_registration => (
14976                "200 OK",
14977                r#"{"worker_id":"declined-worker","registered":false}"#,
14978            ),
14979            "/api/worker/register" if behavior.waiting_query_worker => (
14980                "200 OK",
14981                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14982            ),
14983            "/api/worker/register" => (
14984                "200 OK",
14985                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14986            ),
14987            "/api/worker/heartbeat" => ("200 OK", "{}"),
14988            "/api/worker/activity-tasks/poll"
14989                if behavior.cancelled_activity && request_number == 1 =>
14990            {
14991                (
14992                    "200 OK",
14993                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14994                )
14995            }
14996            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14997                ("200 OK", r#"{"task":null}"#)
14998            }
14999            "/api/worker/query-tasks/poll"
15000                if behavior.reject_query_completion && request_number == 1 =>
15001            {
15002                (
15003                    "200 OK",
15004                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
15005                )
15006            }
15007            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
15008            "/api/worker/query-tasks/query-capture/complete"
15009            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
15010            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
15011                "200 OK",
15012                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
15013            ),
15014            "/api/worker/activity-tasks/activity-cancel/complete" => (
15015                "409 Conflict",
15016                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
15017            ),
15018            "/api/worker/activity-tasks/activity-typed/complete"
15019            | "/api/worker/activity-tasks/activity-typed/fail"
15020            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
15021            "/api/workflows/counter-1/query/current" => (
15022                "200 OK",
15023                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
15024            ),
15025            "/api/workflows/counter-1/query/missing" => (
15026                "404 Not Found",
15027                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
15028            ),
15029            "/api/workflows/wf-lifecycle/cancel" => (
15030                "200 OK",
15031                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
15032            ),
15033            "/api/workflows/wf-lifecycle/terminate" => (
15034                "200 OK",
15035                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
15036            ),
15037            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
15038                "200 OK",
15039                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
15040            ),
15041            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
15042                "200 OK",
15043                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
15044            ),
15045            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
15046            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
15047                "409 Conflict",
15048                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
15049            ),
15050            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
15051                "200 OK",
15052                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
15053            ),
15054            "/api/workflows/wf-cancelled" => (
15055                "200 OK",
15056                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
15057            ),
15058            "/api/workflows/wf-terminated" => (
15059                "200 OK",
15060                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
15061            ),
15062            "/api/workflows/wf-timed-out" => (
15063                "200 OK",
15064                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
15065            ),
15066            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
15067                "200 OK",
15068                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
15069            ),
15070            "/api/workflows/wf-selected" => (
15071                "200 OK",
15072                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
15073            ),
15074            "/api/workflows/wf-selected/runs/run-selected" => (
15075                "200 OK",
15076                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
15077            ),
15078            _ => ("404 Not Found", r#"{"message":"not found"}"#),
15079        };
15080        write_mock_response(stream, status, body);
15081    }
15082
15083    fn mock_request_is_complete(request: &[u8]) -> bool {
15084        let Some(header_end) = request
15085            .windows(4)
15086            .position(|window| window == b"\r\n\r\n")
15087            .map(|position| position + 4)
15088        else {
15089            return false;
15090        };
15091        let headers = String::from_utf8_lossy(&request[..header_end]);
15092        let content_length = headers.lines().find_map(|line| {
15093            let (name, value) = line.split_once(':')?;
15094            name.eq_ignore_ascii_case("content-length")
15095                .then(|| value.trim().parse::<usize>().ok())
15096                .flatten()
15097        });
15098
15099        request.len() >= header_end + content_length.unwrap_or(0)
15100    }
15101
15102    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
15103        let response = format!(
15104            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
15105            body.len()
15106        );
15107
15108        let _ = stream.write_all(response.as_bytes());
15109        let _ = stream.flush();
15110    }
15111}